Commit 05b953a5 authored by Jakub Kicinski's avatar Jakub Kicinski

Merge tag 'mlx5-updates-2023-02-15' of git://git.kernel.org/pub/scm/linux/kernel/git/saeed/linux

Saeed Mahameed says:

====================
mlx5-updates-2023-02-15

1) From Gal Tariq and Parav, Few cleanups for mlx5 driver.

2) From Vlad: Allow offloading of ct 'new' match based on [1]

[1] https://lore.kernel.org/netdev/20230201163100.1001180-1-vladbu@nvidia.com/

* tag 'mlx5-updates-2023-02-15' of git://git.kernel.org/pub/scm/linux/kernel/git/saeed/linux:
  net/mlx5e: RX, Remove doubtful unlikely call
  net/mlx5e: Fix outdated TLS comment
  net/mlx5e: Remove unused function mlx5e_sq_xmit_simple
  net/mlx5e: Allow offloading of ct 'new' match
  net/mlx5e: Implement CT entry update
  net/mlx5: Simplify eq list traversal
  net/mlx5e: Remove redundant page argument in mlx5e_xdp_handle()
  net/mlx5e: Remove redundant page argument in mlx5e_xmit_xdp_buff()
  net/mlx5e: Switch to using napi_build_skb()
====================

Link: https://lore.kernel.org/r/20230218090513.284718-1-saeed@kernel.orgSigned-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parents 981f4045 993fd9bd
...@@ -35,6 +35,7 @@ ...@@ -35,6 +35,7 @@
#define MLX5_CT_STATE_REPLY_BIT BIT(4) #define MLX5_CT_STATE_REPLY_BIT BIT(4)
#define MLX5_CT_STATE_RELATED_BIT BIT(5) #define MLX5_CT_STATE_RELATED_BIT BIT(5)
#define MLX5_CT_STATE_INVALID_BIT BIT(6) #define MLX5_CT_STATE_INVALID_BIT BIT(6)
#define MLX5_CT_STATE_NEW_BIT BIT(7)
#define MLX5_CT_LABELS_BITS MLX5_REG_MAPPING_MBITS(LABELS_TO_REG) #define MLX5_CT_LABELS_BITS MLX5_REG_MAPPING_MBITS(LABELS_TO_REG)
#define MLX5_CT_LABELS_MASK MLX5_REG_MAPPING_MASK(LABELS_TO_REG) #define MLX5_CT_LABELS_MASK MLX5_REG_MAPPING_MASK(LABELS_TO_REG)
...@@ -721,12 +722,14 @@ mlx5_tc_ct_entry_create_mod_hdr(struct mlx5_tc_ct_priv *ct_priv, ...@@ -721,12 +722,14 @@ mlx5_tc_ct_entry_create_mod_hdr(struct mlx5_tc_ct_priv *ct_priv,
DECLARE_MOD_HDR_ACTS_ACTIONS(actions_arr, MLX5_CT_MIN_MOD_ACTS); DECLARE_MOD_HDR_ACTS_ACTIONS(actions_arr, MLX5_CT_MIN_MOD_ACTS);
DECLARE_MOD_HDR_ACTS(mod_acts, actions_arr); DECLARE_MOD_HDR_ACTS(mod_acts, actions_arr);
struct flow_action_entry *meta; struct flow_action_entry *meta;
enum ip_conntrack_info ctinfo;
u16 ct_state = 0; u16 ct_state = 0;
int err; int err;
meta = mlx5_tc_ct_get_ct_metadata_action(flow_rule); meta = mlx5_tc_ct_get_ct_metadata_action(flow_rule);
if (!meta) if (!meta)
return -EOPNOTSUPP; return -EOPNOTSUPP;
ctinfo = meta->ct_metadata.cookie & NFCT_INFOMASK;
err = mlx5_get_label_mapping(ct_priv, meta->ct_metadata.labels, err = mlx5_get_label_mapping(ct_priv, meta->ct_metadata.labels,
&attr->ct_attr.ct_labels_id); &attr->ct_attr.ct_labels_id);
...@@ -742,7 +745,8 @@ mlx5_tc_ct_entry_create_mod_hdr(struct mlx5_tc_ct_priv *ct_priv, ...@@ -742,7 +745,8 @@ mlx5_tc_ct_entry_create_mod_hdr(struct mlx5_tc_ct_priv *ct_priv,
ct_state |= MLX5_CT_STATE_NAT_BIT; ct_state |= MLX5_CT_STATE_NAT_BIT;
} }
ct_state |= MLX5_CT_STATE_ESTABLISHED_BIT | MLX5_CT_STATE_TRK_BIT; ct_state |= MLX5_CT_STATE_TRK_BIT;
ct_state |= ctinfo == IP_CT_NEW ? MLX5_CT_STATE_NEW_BIT : MLX5_CT_STATE_ESTABLISHED_BIT;
ct_state |= meta->ct_metadata.orig_dir ? 0 : MLX5_CT_STATE_REPLY_BIT; ct_state |= meta->ct_metadata.orig_dir ? 0 : MLX5_CT_STATE_REPLY_BIT;
err = mlx5_tc_ct_entry_set_registers(ct_priv, &mod_acts, err = mlx5_tc_ct_entry_set_registers(ct_priv, &mod_acts,
ct_state, ct_state,
...@@ -871,6 +875,68 @@ mlx5_tc_ct_entry_add_rule(struct mlx5_tc_ct_priv *ct_priv, ...@@ -871,6 +875,68 @@ mlx5_tc_ct_entry_add_rule(struct mlx5_tc_ct_priv *ct_priv,
return err; return err;
} }
static int
mlx5_tc_ct_entry_replace_rule(struct mlx5_tc_ct_priv *ct_priv,
struct flow_rule *flow_rule,
struct mlx5_ct_entry *entry,
bool nat, u8 zone_restore_id)
{
struct mlx5_ct_zone_rule *zone_rule = &entry->zone_rules[nat];
struct mlx5_flow_attr *attr = zone_rule->attr, *old_attr;
struct mlx5e_mod_hdr_handle *mh;
struct mlx5_ct_fs_rule *rule;
struct mlx5_flow_spec *spec;
int err;
spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
if (!spec)
return -ENOMEM;
old_attr = mlx5_alloc_flow_attr(ct_priv->ns_type);
if (!old_attr) {
err = -ENOMEM;
goto err_attr;
}
*old_attr = *attr;
err = mlx5_tc_ct_entry_create_mod_hdr(ct_priv, attr, flow_rule, &mh, zone_restore_id,
nat, mlx5_tc_ct_entry_has_nat(entry));
if (err) {
ct_dbg("Failed to create ct entry mod hdr");
goto err_mod_hdr;
}
mlx5_tc_ct_set_tuple_match(ct_priv, spec, flow_rule);
mlx5e_tc_match_to_reg_match(spec, ZONE_TO_REG, entry->tuple.zone, MLX5_CT_ZONE_MASK);
rule = ct_priv->fs_ops->ct_rule_add(ct_priv->fs, spec, attr, flow_rule);
if (IS_ERR(rule)) {
err = PTR_ERR(rule);
ct_dbg("Failed to add replacement ct entry rule, nat: %d", nat);
goto err_rule;
}
ct_priv->fs_ops->ct_rule_del(ct_priv->fs, zone_rule->rule);
zone_rule->rule = rule;
mlx5_tc_ct_entry_destroy_mod_hdr(ct_priv, old_attr, zone_rule->mh);
zone_rule->mh = mh;
kfree(old_attr);
kvfree(spec);
ct_dbg("Replaced ct entry rule in zone %d", entry->tuple.zone);
return 0;
err_rule:
mlx5_tc_ct_entry_destroy_mod_hdr(ct_priv, zone_rule->attr, mh);
mlx5_put_label_mapping(ct_priv, attr->ct_attr.ct_labels_id);
err_mod_hdr:
kfree(old_attr);
err_attr:
kvfree(spec);
return err;
}
static bool static bool
mlx5_tc_ct_entry_valid(struct mlx5_ct_entry *entry) mlx5_tc_ct_entry_valid(struct mlx5_ct_entry *entry)
{ {
...@@ -1065,6 +1131,52 @@ mlx5_tc_ct_entry_add_rules(struct mlx5_tc_ct_priv *ct_priv, ...@@ -1065,6 +1131,52 @@ mlx5_tc_ct_entry_add_rules(struct mlx5_tc_ct_priv *ct_priv,
return err; return err;
} }
static int
mlx5_tc_ct_entry_replace_rules(struct mlx5_tc_ct_priv *ct_priv,
struct flow_rule *flow_rule,
struct mlx5_ct_entry *entry,
u8 zone_restore_id)
{
int err;
err = mlx5_tc_ct_entry_replace_rule(ct_priv, flow_rule, entry, false,
zone_restore_id);
if (err)
return err;
err = mlx5_tc_ct_entry_replace_rule(ct_priv, flow_rule, entry, true,
zone_restore_id);
if (err)
mlx5_tc_ct_entry_del_rule(ct_priv, entry, false);
return err;
}
static int
mlx5_tc_ct_block_flow_offload_replace(struct mlx5_ct_ft *ft, struct flow_rule *flow_rule,
struct mlx5_ct_entry *entry, unsigned long cookie)
{
struct mlx5_tc_ct_priv *ct_priv = ft->ct_priv;
int err;
err = mlx5_tc_ct_entry_replace_rules(ct_priv, flow_rule, entry, ft->zone_restore_id);
if (!err)
return 0;
/* If failed to update the entry, then look it up again under ht_lock
* protection and properly delete it.
*/
spin_lock_bh(&ct_priv->ht_lock);
entry = rhashtable_lookup_fast(&ft->ct_entries_ht, &cookie, cts_ht_params);
if (entry) {
rhashtable_remove_fast(&ft->ct_entries_ht, &entry->node, cts_ht_params);
spin_unlock_bh(&ct_priv->ht_lock);
mlx5_tc_ct_entry_put(entry);
} else {
spin_unlock_bh(&ct_priv->ht_lock);
}
return err;
}
static int static int
mlx5_tc_ct_block_flow_offload_add(struct mlx5_ct_ft *ft, mlx5_tc_ct_block_flow_offload_add(struct mlx5_ct_ft *ft,
struct flow_cls_offload *flow) struct flow_cls_offload *flow)
...@@ -1073,23 +1185,27 @@ mlx5_tc_ct_block_flow_offload_add(struct mlx5_ct_ft *ft, ...@@ -1073,23 +1185,27 @@ mlx5_tc_ct_block_flow_offload_add(struct mlx5_ct_ft *ft,
struct mlx5_tc_ct_priv *ct_priv = ft->ct_priv; struct mlx5_tc_ct_priv *ct_priv = ft->ct_priv;
struct flow_action_entry *meta_action; struct flow_action_entry *meta_action;
unsigned long cookie = flow->cookie; unsigned long cookie = flow->cookie;
enum ip_conntrack_info ctinfo;
struct mlx5_ct_entry *entry; struct mlx5_ct_entry *entry;
int err; int err;
meta_action = mlx5_tc_ct_get_ct_metadata_action(flow_rule); meta_action = mlx5_tc_ct_get_ct_metadata_action(flow_rule);
if (!meta_action) if (!meta_action)
return -EOPNOTSUPP; return -EOPNOTSUPP;
ctinfo = meta_action->ct_metadata.cookie & NFCT_INFOMASK;
if (ctinfo == IP_CT_NEW)
return -EOPNOTSUPP;
spin_lock_bh(&ct_priv->ht_lock); spin_lock_bh(&ct_priv->ht_lock);
entry = rhashtable_lookup_fast(&ft->ct_entries_ht, &cookie, cts_ht_params); entry = rhashtable_lookup_fast(&ft->ct_entries_ht, &cookie, cts_ht_params);
if (entry && refcount_inc_not_zero(&entry->refcnt)) { if (entry && refcount_inc_not_zero(&entry->refcnt)) {
if (entry->restore_cookie == meta_action->ct_metadata.cookie) {
spin_unlock_bh(&ct_priv->ht_lock);
mlx5_tc_ct_entry_put(entry);
return -EEXIST;
}
entry->restore_cookie = meta_action->ct_metadata.cookie;
spin_unlock_bh(&ct_priv->ht_lock); spin_unlock_bh(&ct_priv->ht_lock);
err = mlx5_tc_ct_block_flow_offload_replace(ft, flow_rule, entry, cookie);
mlx5_tc_ct_entry_put(entry); mlx5_tc_ct_entry_put(entry);
return -EEXIST; return err;
} }
spin_unlock_bh(&ct_priv->ht_lock); spin_unlock_bh(&ct_priv->ht_lock);
...@@ -1327,7 +1443,7 @@ mlx5_tc_ct_match_add(struct mlx5_tc_ct_priv *priv, ...@@ -1327,7 +1443,7 @@ mlx5_tc_ct_match_add(struct mlx5_tc_ct_priv *priv,
struct mlx5_ct_attr *ct_attr, struct mlx5_ct_attr *ct_attr,
struct netlink_ext_ack *extack) struct netlink_ext_ack *extack)
{ {
bool trk, est, untrk, unest, new, rpl, unrpl, rel, unrel, inv, uninv; bool trk, est, untrk, unnew, unest, new, rpl, unrpl, rel, unrel, inv, uninv;
struct flow_rule *rule = flow_cls_offload_flow_rule(f); struct flow_rule *rule = flow_cls_offload_flow_rule(f);
struct flow_dissector_key_ct *mask, *key; struct flow_dissector_key_ct *mask, *key;
u32 ctstate = 0, ctstate_mask = 0; u32 ctstate = 0, ctstate_mask = 0;
...@@ -1373,15 +1489,18 @@ mlx5_tc_ct_match_add(struct mlx5_tc_ct_priv *priv, ...@@ -1373,15 +1489,18 @@ mlx5_tc_ct_match_add(struct mlx5_tc_ct_priv *priv,
rel = ct_state_on & TCA_FLOWER_KEY_CT_FLAGS_RELATED; rel = ct_state_on & TCA_FLOWER_KEY_CT_FLAGS_RELATED;
inv = ct_state_on & TCA_FLOWER_KEY_CT_FLAGS_INVALID; inv = ct_state_on & TCA_FLOWER_KEY_CT_FLAGS_INVALID;
untrk = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_TRACKED; untrk = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_TRACKED;
unnew = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_NEW;
unest = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED; unest = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED;
unrpl = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_REPLY; unrpl = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_REPLY;
unrel = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_RELATED; unrel = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_RELATED;
uninv = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_INVALID; uninv = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_INVALID;
ctstate |= trk ? MLX5_CT_STATE_TRK_BIT : 0; ctstate |= trk ? MLX5_CT_STATE_TRK_BIT : 0;
ctstate |= new ? MLX5_CT_STATE_NEW_BIT : 0;
ctstate |= est ? MLX5_CT_STATE_ESTABLISHED_BIT : 0; ctstate |= est ? MLX5_CT_STATE_ESTABLISHED_BIT : 0;
ctstate |= rpl ? MLX5_CT_STATE_REPLY_BIT : 0; ctstate |= rpl ? MLX5_CT_STATE_REPLY_BIT : 0;
ctstate_mask |= (untrk || trk) ? MLX5_CT_STATE_TRK_BIT : 0; ctstate_mask |= (untrk || trk) ? MLX5_CT_STATE_TRK_BIT : 0;
ctstate_mask |= (unnew || new) ? MLX5_CT_STATE_NEW_BIT : 0;
ctstate_mask |= (unest || est) ? MLX5_CT_STATE_ESTABLISHED_BIT : 0; ctstate_mask |= (unest || est) ? MLX5_CT_STATE_ESTABLISHED_BIT : 0;
ctstate_mask |= (unrpl || rpl) ? MLX5_CT_STATE_REPLY_BIT : 0; ctstate_mask |= (unrpl || rpl) ? MLX5_CT_STATE_REPLY_BIT : 0;
ctstate_mask |= unrel ? MLX5_CT_STATE_RELATED_BIT : 0; ctstate_mask |= unrel ? MLX5_CT_STATE_RELATED_BIT : 0;
...@@ -1399,12 +1518,6 @@ mlx5_tc_ct_match_add(struct mlx5_tc_ct_priv *priv, ...@@ -1399,12 +1518,6 @@ mlx5_tc_ct_match_add(struct mlx5_tc_ct_priv *priv,
return -EOPNOTSUPP; return -EOPNOTSUPP;
} }
if (new) {
NL_SET_ERR_MSG_MOD(extack,
"matching on ct_state +new isn't supported");
return -EOPNOTSUPP;
}
if (mask->ct_zone) if (mask->ct_zone)
mlx5e_tc_match_to_reg_match(spec, ZONE_TO_REG, mlx5e_tc_match_to_reg_match(spec, ZONE_TO_REG,
key->ct_zone, MLX5_CT_ZONE_MASK); key->ct_zone, MLX5_CT_ZONE_MASK);
......
...@@ -320,7 +320,6 @@ mlx5e_tx_dma_unmap(struct device *pdev, struct mlx5e_sq_dma *dma) ...@@ -320,7 +320,6 @@ mlx5e_tx_dma_unmap(struct device *pdev, struct mlx5e_sq_dma *dma)
} }
} }
void mlx5e_sq_xmit_simple(struct mlx5e_txqsq *sq, struct sk_buff *skb, bool xmit_more);
void mlx5e_tx_mpwqe_ensure_complete(struct mlx5e_txqsq *sq); void mlx5e_tx_mpwqe_ensure_complete(struct mlx5e_txqsq *sq);
static inline bool mlx5e_tx_mpwqe_is_full(struct mlx5e_tx_mpwqe *session, u8 max_sq_mpw_wqebbs) static inline bool mlx5e_tx_mpwqe_is_full(struct mlx5e_tx_mpwqe *session, u8 max_sq_mpw_wqebbs)
......
...@@ -57,8 +57,9 @@ int mlx5e_xdp_max_mtu(struct mlx5e_params *params, struct mlx5e_xsk_param *xsk) ...@@ -57,8 +57,9 @@ int mlx5e_xdp_max_mtu(struct mlx5e_params *params, struct mlx5e_xsk_param *xsk)
static inline bool static inline bool
mlx5e_xmit_xdp_buff(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq, mlx5e_xmit_xdp_buff(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq,
struct page *page, struct xdp_buff *xdp) struct xdp_buff *xdp)
{ {
struct page *page = virt_to_page(xdp->data);
struct skb_shared_info *sinfo = NULL; struct skb_shared_info *sinfo = NULL;
struct mlx5e_xmit_data xdptxd; struct mlx5e_xmit_data xdptxd;
struct mlx5e_xdp_info xdpi; struct mlx5e_xdp_info xdpi;
...@@ -185,7 +186,7 @@ const struct xdp_metadata_ops mlx5e_xdp_metadata_ops = { ...@@ -185,7 +186,7 @@ const struct xdp_metadata_ops mlx5e_xdp_metadata_ops = {
}; };
/* returns true if packet was consumed by xdp */ /* returns true if packet was consumed by xdp */
bool mlx5e_xdp_handle(struct mlx5e_rq *rq, struct page *page, bool mlx5e_xdp_handle(struct mlx5e_rq *rq,
struct bpf_prog *prog, struct mlx5e_xdp_buff *mxbuf) struct bpf_prog *prog, struct mlx5e_xdp_buff *mxbuf)
{ {
struct xdp_buff *xdp = &mxbuf->xdp; struct xdp_buff *xdp = &mxbuf->xdp;
...@@ -197,7 +198,7 @@ bool mlx5e_xdp_handle(struct mlx5e_rq *rq, struct page *page, ...@@ -197,7 +198,7 @@ bool mlx5e_xdp_handle(struct mlx5e_rq *rq, struct page *page,
case XDP_PASS: case XDP_PASS:
return false; return false;
case XDP_TX: case XDP_TX:
if (unlikely(!mlx5e_xmit_xdp_buff(rq->xdpsq, rq, page, xdp))) if (unlikely(!mlx5e_xmit_xdp_buff(rq->xdpsq, rq, xdp)))
goto xdp_abort; goto xdp_abort;
__set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags); /* non-atomic */ __set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags); /* non-atomic */
return true; return true;
...@@ -209,7 +210,7 @@ bool mlx5e_xdp_handle(struct mlx5e_rq *rq, struct page *page, ...@@ -209,7 +210,7 @@ bool mlx5e_xdp_handle(struct mlx5e_rq *rq, struct page *page,
__set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags); __set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags);
__set_bit(MLX5E_RQ_FLAG_XDP_REDIRECT, rq->flags); __set_bit(MLX5E_RQ_FLAG_XDP_REDIRECT, rq->flags);
if (xdp->rxq->mem.type != MEM_TYPE_XSK_BUFF_POOL) if (xdp->rxq->mem.type != MEM_TYPE_XSK_BUFF_POOL)
mlx5e_page_dma_unmap(rq, page); mlx5e_page_dma_unmap(rq, virt_to_page(xdp->data));
rq->stats->xdp_redirect++; rq->stats->xdp_redirect++;
return true; return true;
default: default:
......
...@@ -52,7 +52,7 @@ struct mlx5e_xdp_buff { ...@@ -52,7 +52,7 @@ struct mlx5e_xdp_buff {
struct mlx5e_xsk_param; struct mlx5e_xsk_param;
int mlx5e_xdp_max_mtu(struct mlx5e_params *params, struct mlx5e_xsk_param *xsk); int mlx5e_xdp_max_mtu(struct mlx5e_params *params, struct mlx5e_xsk_param *xsk);
bool mlx5e_xdp_handle(struct mlx5e_rq *rq, struct page *page, bool mlx5e_xdp_handle(struct mlx5e_rq *rq,
struct bpf_prog *prog, struct mlx5e_xdp_buff *mlctx); struct bpf_prog *prog, struct mlx5e_xdp_buff *mlctx);
void mlx5e_xdp_mpwqe_complete(struct mlx5e_xdpsq *sq); void mlx5e_xdp_mpwqe_complete(struct mlx5e_xdpsq *sq);
bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq); bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq);
......
...@@ -289,7 +289,7 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, ...@@ -289,7 +289,7 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq,
*/ */
prog = rcu_dereference(rq->xdp_prog); prog = rcu_dereference(rq->xdp_prog);
if (likely(prog && mlx5e_xdp_handle(rq, NULL, prog, mxbuf))) { if (likely(prog && mlx5e_xdp_handle(rq, prog, mxbuf))) {
if (likely(__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags))) if (likely(__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)))
__set_bit(page_idx, wi->xdp_xmit_bitmap); /* non-atomic */ __set_bit(page_idx, wi->xdp_xmit_bitmap); /* non-atomic */
return NULL; /* page/packet was consumed by XDP */ return NULL; /* page/packet was consumed by XDP */
...@@ -323,7 +323,7 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq, ...@@ -323,7 +323,7 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq,
net_prefetch(mxbuf->xdp.data); net_prefetch(mxbuf->xdp.data);
prog = rcu_dereference(rq->xdp_prog); prog = rcu_dereference(rq->xdp_prog);
if (likely(prog && mlx5e_xdp_handle(rq, NULL, prog, mxbuf))) if (likely(prog && mlx5e_xdp_handle(rq, prog, mxbuf)))
return NULL; /* page/packet was consumed by XDP */ return NULL; /* page/packet was consumed by XDP */
/* XDP_PASS: copy the data from the UMEM to a new SKB. The frame reuse /* XDP_PASS: copy the data from the UMEM to a new SKB. The frame reuse
......
...@@ -124,7 +124,7 @@ static inline bool mlx5e_accel_tx_begin(struct net_device *dev, ...@@ -124,7 +124,7 @@ static inline bool mlx5e_accel_tx_begin(struct net_device *dev,
mlx5e_udp_gso_handle_tx_skb(skb); mlx5e_udp_gso_handle_tx_skb(skb);
#ifdef CONFIG_MLX5_EN_TLS #ifdef CONFIG_MLX5_EN_TLS
/* May send SKBs and WQEs. */ /* May send WQEs. */
if (mlx5e_ktls_skb_offloaded(skb)) if (mlx5e_ktls_skb_offloaded(skb))
if (unlikely(!mlx5e_ktls_handle_tx_skb(dev, sq, skb, if (unlikely(!mlx5e_ktls_handle_tx_skb(dev, sq, skb,
&state->tls))) &state->tls)))
......
...@@ -1556,7 +1556,7 @@ struct sk_buff *mlx5e_build_linear_skb(struct mlx5e_rq *rq, void *va, ...@@ -1556,7 +1556,7 @@ struct sk_buff *mlx5e_build_linear_skb(struct mlx5e_rq *rq, void *va,
u32 frag_size, u16 headroom, u32 frag_size, u16 headroom,
u32 cqe_bcnt, u32 metasize) u32 cqe_bcnt, u32 metasize)
{ {
struct sk_buff *skb = build_skb(va, frag_size); struct sk_buff *skb = napi_build_skb(va, frag_size);
if (unlikely(!skb)) { if (unlikely(!skb)) {
rq->stats->buff_alloc_err++; rq->stats->buff_alloc_err++;
...@@ -1610,7 +1610,7 @@ mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi, ...@@ -1610,7 +1610,7 @@ mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi,
net_prefetchw(va); /* xdp_frame data area */ net_prefetchw(va); /* xdp_frame data area */
mlx5e_fill_mxbuf(rq, cqe, va, rx_headroom, cqe_bcnt, &mxbuf); mlx5e_fill_mxbuf(rq, cqe, va, rx_headroom, cqe_bcnt, &mxbuf);
if (mlx5e_xdp_handle(rq, au->page, prog, &mxbuf)) if (mlx5e_xdp_handle(rq, prog, &mxbuf))
return NULL; /* page/packet was consumed by XDP */ return NULL; /* page/packet was consumed by XDP */
rx_headroom = mxbuf.xdp.data - mxbuf.xdp.data_hard_start; rx_headroom = mxbuf.xdp.data - mxbuf.xdp.data_hard_start;
...@@ -1698,10 +1698,8 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi ...@@ -1698,10 +1698,8 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi
wi++; wi++;
} }
au = head_wi->au;
prog = rcu_dereference(rq->xdp_prog); prog = rcu_dereference(rq->xdp_prog);
if (prog && mlx5e_xdp_handle(rq, au->page, prog, &mxbuf)) { if (prog && mlx5e_xdp_handle(rq, prog, &mxbuf)) {
if (test_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)) { if (test_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)) {
int i; int i;
...@@ -1718,9 +1716,9 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi ...@@ -1718,9 +1716,9 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi
if (unlikely(!skb)) if (unlikely(!skb))
return NULL; return NULL;
page_ref_inc(au->page); page_ref_inc(head_wi->au->page);
if (unlikely(xdp_buff_has_frags(&mxbuf.xdp))) { if (xdp_buff_has_frags(&mxbuf.xdp)) {
int i; int i;
/* sinfo->nr_frags is reset by build_skb, calculate again. */ /* sinfo->nr_frags is reset by build_skb, calculate again. */
...@@ -2013,7 +2011,7 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, ...@@ -2013,7 +2011,7 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
net_prefetchw(va); /* xdp_frame data area */ net_prefetchw(va); /* xdp_frame data area */
mlx5e_fill_mxbuf(rq, cqe, va, rx_headroom, cqe_bcnt, &mxbuf); mlx5e_fill_mxbuf(rq, cqe, va, rx_headroom, cqe_bcnt, &mxbuf);
if (mlx5e_xdp_handle(rq, au->page, prog, &mxbuf)) { if (mlx5e_xdp_handle(rq, prog, &mxbuf)) {
if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)) if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags))
__set_bit(page_idx, wi->xdp_xmit_bitmap); /* non-atomic */ __set_bit(page_idx, wi->xdp_xmit_bitmap); /* non-atomic */
return NULL; /* page/packet was consumed by XDP */ return NULL; /* page/packet was consumed by XDP */
......
...@@ -720,21 +720,6 @@ netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev) ...@@ -720,21 +720,6 @@ netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev)
return NETDEV_TX_OK; return NETDEV_TX_OK;
} }
void mlx5e_sq_xmit_simple(struct mlx5e_txqsq *sq, struct sk_buff *skb, bool xmit_more)
{
struct mlx5e_tx_wqe_attr wqe_attr;
struct mlx5e_tx_attr attr;
struct mlx5e_tx_wqe *wqe;
u16 pi;
mlx5e_sq_xmit_prepare(sq, skb, NULL, &attr);
mlx5e_sq_calc_wqe_attr(skb, &attr, &wqe_attr);
pi = mlx5e_txqsq_get_next_pi(sq, wqe_attr.num_wqebbs);
wqe = MLX5E_TX_FETCH_WQE(sq, pi);
mlx5e_txwqe_build_eseg_csum(sq, skb, NULL, &wqe->eth);
mlx5e_sq_xmit_wqe(sq, skb, &attr, &wqe_attr, wqe, pi, xmit_more);
}
static void mlx5e_tx_wi_dma_unmap(struct mlx5e_txqsq *sq, struct mlx5e_tx_wqe_info *wi, static void mlx5e_tx_wi_dma_unmap(struct mlx5e_txqsq *sq, struct mlx5e_tx_wqe_info *wi,
u32 *dma_fifo_cc) u32 *dma_fifo_cc)
{ {
......
...@@ -960,11 +960,11 @@ static int vector2eqnirqn(struct mlx5_core_dev *dev, int vector, int *eqn, ...@@ -960,11 +960,11 @@ static int vector2eqnirqn(struct mlx5_core_dev *dev, int vector, int *eqn,
unsigned int *irqn) unsigned int *irqn)
{ {
struct mlx5_eq_table *table = dev->priv.eq_table; struct mlx5_eq_table *table = dev->priv.eq_table;
struct mlx5_eq_comp *eq, *n; struct mlx5_eq_comp *eq;
int err = -ENOENT; int err = -ENOENT;
int i = 0; int i = 0;
list_for_each_entry_safe(eq, n, &table->comp_eqs_list, list) { list_for_each_entry(eq, &table->comp_eqs_list, list) {
if (i++ == vector) { if (i++ == vector) {
if (irqn) if (irqn)
*irqn = eq->core.irqn; *irqn = eq->core.irqn;
...@@ -999,10 +999,10 @@ struct cpumask * ...@@ -999,10 +999,10 @@ struct cpumask *
mlx5_comp_irq_get_affinity_mask(struct mlx5_core_dev *dev, int vector) mlx5_comp_irq_get_affinity_mask(struct mlx5_core_dev *dev, int vector)
{ {
struct mlx5_eq_table *table = dev->priv.eq_table; struct mlx5_eq_table *table = dev->priv.eq_table;
struct mlx5_eq_comp *eq, *n; struct mlx5_eq_comp *eq;
int i = 0; int i = 0;
list_for_each_entry_safe(eq, n, &table->comp_eqs_list, list) { list_for_each_entry(eq, &table->comp_eqs_list, list) {
if (i++ == vector) if (i++ == vector)
break; break;
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment