Commit 6e144b47 authored by Suman Ghosh's avatar Suman Ghosh Committed by Jakub Kicinski

octeontx2-pf: Add support for adaptive interrupt coalescing

Added support for adaptive IRQ coalescing. It uses net_dim
algorithm to find the suitable delay/IRQ count based on the
current packet rate.
Signed-off-by: default avatarSuman Ghosh <sumang@marvell.com>
Link: https://lore.kernel.org/r/20220517044055.876158-1-sumang@marvell.comSigned-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parent 9cc34128
...@@ -33,6 +33,7 @@ config OCTEONTX2_PF ...@@ -33,6 +33,7 @@ config OCTEONTX2_PF
select OCTEONTX2_MBOX select OCTEONTX2_MBOX
select NET_DEVLINK select NET_DEVLINK
depends on (64BIT && COMPILE_TEST) || ARM64 depends on (64BIT && COMPILE_TEST) || ARM64
select DIMLIB
depends on PCI depends on PCI
depends on PTP_1588_CLOCK_OPTIONAL depends on PTP_1588_CLOCK_OPTIONAL
help help
......
...@@ -97,11 +97,6 @@ void otx2_get_dev_stats(struct otx2_nic *pfvf) ...@@ -97,11 +97,6 @@ void otx2_get_dev_stats(struct otx2_nic *pfvf)
{ {
struct otx2_dev_stats *dev_stats = &pfvf->hw.dev_stats; struct otx2_dev_stats *dev_stats = &pfvf->hw.dev_stats;
#define OTX2_GET_RX_STATS(reg) \
otx2_read64(pfvf, NIX_LF_RX_STATX(reg))
#define OTX2_GET_TX_STATS(reg) \
otx2_read64(pfvf, NIX_LF_TX_STATX(reg))
dev_stats->rx_bytes = OTX2_GET_RX_STATS(RX_OCTS); dev_stats->rx_bytes = OTX2_GET_RX_STATS(RX_OCTS);
dev_stats->rx_drops = OTX2_GET_RX_STATS(RX_DROP); dev_stats->rx_drops = OTX2_GET_RX_STATS(RX_DROP);
dev_stats->rx_bcast_frames = OTX2_GET_RX_STATS(RX_BCAST); dev_stats->rx_bcast_frames = OTX2_GET_RX_STATS(RX_BCAST);
......
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
#include <net/pkt_cls.h> #include <net/pkt_cls.h>
#include <net/devlink.h> #include <net/devlink.h>
#include <linux/time64.h> #include <linux/time64.h>
#include <linux/dim.h>
#include <mbox.h> #include <mbox.h>
#include <npc.h> #include <npc.h>
...@@ -54,6 +55,11 @@ enum arua_mapped_qtypes { ...@@ -54,6 +55,11 @@ enum arua_mapped_qtypes {
/* Send skid of 2000 packets required for CQ size of 4K CQEs. */ /* Send skid of 2000 packets required for CQ size of 4K CQEs. */
#define SEND_CQ_SKID 2000 #define SEND_CQ_SKID 2000
#define OTX2_GET_RX_STATS(reg) \
otx2_read64(pfvf, NIX_LF_RX_STATX(reg))
#define OTX2_GET_TX_STATS(reg) \
otx2_read64(pfvf, NIX_LF_TX_STATX(reg))
struct otx2_lmt_info { struct otx2_lmt_info {
u64 lmt_addr; u64 lmt_addr;
u16 lmt_id; u16 lmt_id;
...@@ -351,6 +357,7 @@ struct otx2_nic { ...@@ -351,6 +357,7 @@ struct otx2_nic {
#define OTX2_FLAG_TC_MATCHALL_EGRESS_ENABLED BIT_ULL(12) #define OTX2_FLAG_TC_MATCHALL_EGRESS_ENABLED BIT_ULL(12)
#define OTX2_FLAG_TC_MATCHALL_INGRESS_ENABLED BIT_ULL(13) #define OTX2_FLAG_TC_MATCHALL_INGRESS_ENABLED BIT_ULL(13)
#define OTX2_FLAG_DMACFLTR_SUPPORT BIT_ULL(14) #define OTX2_FLAG_DMACFLTR_SUPPORT BIT_ULL(14)
#define OTX2_FLAG_ADPTV_INT_COAL_ENABLED BIT_ULL(16)
u64 flags; u64 flags;
u64 *cq_op_addr; u64 *cq_op_addr;
...@@ -408,6 +415,9 @@ struct otx2_nic { ...@@ -408,6 +415,9 @@ struct otx2_nic {
u8 pfc_en; u8 pfc_en;
u8 *queue_to_pfc_map; u8 *queue_to_pfc_map;
#endif #endif
/* napi event count. It is needed for adaptive irq coalescing. */
u32 napi_events;
}; };
static inline bool is_otx2_lbkvf(struct pci_dev *pdev) static inline bool is_otx2_lbkvf(struct pci_dev *pdev)
......
...@@ -455,6 +455,14 @@ static int otx2_get_coalesce(struct net_device *netdev, ...@@ -455,6 +455,14 @@ static int otx2_get_coalesce(struct net_device *netdev,
cmd->rx_max_coalesced_frames = hw->cq_ecount_wait; cmd->rx_max_coalesced_frames = hw->cq_ecount_wait;
cmd->tx_coalesce_usecs = hw->cq_time_wait; cmd->tx_coalesce_usecs = hw->cq_time_wait;
cmd->tx_max_coalesced_frames = hw->cq_ecount_wait; cmd->tx_max_coalesced_frames = hw->cq_ecount_wait;
if ((pfvf->flags & OTX2_FLAG_ADPTV_INT_COAL_ENABLED) ==
OTX2_FLAG_ADPTV_INT_COAL_ENABLED) {
cmd->use_adaptive_rx_coalesce = 1;
cmd->use_adaptive_tx_coalesce = 1;
} else {
cmd->use_adaptive_rx_coalesce = 0;
cmd->use_adaptive_tx_coalesce = 0;
}
return 0; return 0;
} }
...@@ -466,11 +474,30 @@ static int otx2_set_coalesce(struct net_device *netdev, ...@@ -466,11 +474,30 @@ static int otx2_set_coalesce(struct net_device *netdev,
{ {
struct otx2_nic *pfvf = netdev_priv(netdev); struct otx2_nic *pfvf = netdev_priv(netdev);
struct otx2_hw *hw = &pfvf->hw; struct otx2_hw *hw = &pfvf->hw;
u8 priv_coalesce_status;
int qidx; int qidx;
if (!ec->rx_max_coalesced_frames || !ec->tx_max_coalesced_frames) if (!ec->rx_max_coalesced_frames || !ec->tx_max_coalesced_frames)
return 0; return 0;
if (ec->use_adaptive_rx_coalesce != ec->use_adaptive_tx_coalesce) {
netdev_err(netdev,
"adaptive-rx should be same as adaptive-tx");
return -EINVAL;
}
/* Check and update coalesce status */
if ((pfvf->flags & OTX2_FLAG_ADPTV_INT_COAL_ENABLED) ==
OTX2_FLAG_ADPTV_INT_COAL_ENABLED) {
priv_coalesce_status = 1;
if (!ec->use_adaptive_rx_coalesce)
pfvf->flags &= ~OTX2_FLAG_ADPTV_INT_COAL_ENABLED;
} else {
priv_coalesce_status = 0;
if (ec->use_adaptive_rx_coalesce)
pfvf->flags |= OTX2_FLAG_ADPTV_INT_COAL_ENABLED;
}
/* 'cq_time_wait' is 8bit and is in multiple of 100ns, /* 'cq_time_wait' is 8bit and is in multiple of 100ns,
* so clamp the user given value to the range of 1 to 25usec. * so clamp the user given value to the range of 1 to 25usec.
*/ */
...@@ -494,9 +521,9 @@ static int otx2_set_coalesce(struct net_device *netdev, ...@@ -494,9 +521,9 @@ static int otx2_set_coalesce(struct net_device *netdev,
* so clamp the user given value to the range of 1 to 64k. * so clamp the user given value to the range of 1 to 64k.
*/ */
ec->rx_max_coalesced_frames = clamp_t(u32, ec->rx_max_coalesced_frames, ec->rx_max_coalesced_frames = clamp_t(u32, ec->rx_max_coalesced_frames,
1, U16_MAX); 1, NAPI_POLL_WEIGHT);
ec->tx_max_coalesced_frames = clamp_t(u32, ec->tx_max_coalesced_frames, ec->tx_max_coalesced_frames = clamp_t(u32, ec->tx_max_coalesced_frames,
1, U16_MAX); 1, NAPI_POLL_WEIGHT);
/* Rx and Tx are mapped to same CQ, check which one /* Rx and Tx are mapped to same CQ, check which one
* is changed, if both then choose the min. * is changed, if both then choose the min.
...@@ -509,6 +536,17 @@ static int otx2_set_coalesce(struct net_device *netdev, ...@@ -509,6 +536,17 @@ static int otx2_set_coalesce(struct net_device *netdev,
hw->cq_ecount_wait = min_t(u16, ec->rx_max_coalesced_frames, hw->cq_ecount_wait = min_t(u16, ec->rx_max_coalesced_frames,
ec->tx_max_coalesced_frames); ec->tx_max_coalesced_frames);
/* Reset 'cq_time_wait' and 'cq_ecount_wait' to
* default values if coalesce status changed from
* 'on' to 'off'.
*/
if (priv_coalesce_status &&
((pfvf->flags & OTX2_FLAG_ADPTV_INT_COAL_ENABLED) !=
OTX2_FLAG_ADPTV_INT_COAL_ENABLED)) {
hw->cq_time_wait = CQ_TIMER_THRESH_DEFAULT;
hw->cq_ecount_wait = CQ_CQE_THRESH_DEFAULT;
}
if (netif_running(netdev)) { if (netif_running(netdev)) {
for (qidx = 0; qidx < pfvf->hw.cint_cnt; qidx++) for (qidx = 0; qidx < pfvf->hw.cint_cnt; qidx++)
otx2_config_irq_coalescing(pfvf, qidx); otx2_config_irq_coalescing(pfvf, qidx);
...@@ -1230,7 +1268,8 @@ static int otx2_set_link_ksettings(struct net_device *netdev, ...@@ -1230,7 +1268,8 @@ static int otx2_set_link_ksettings(struct net_device *netdev,
static const struct ethtool_ops otx2_ethtool_ops = { static const struct ethtool_ops otx2_ethtool_ops = {
.supported_coalesce_params = ETHTOOL_COALESCE_USECS | .supported_coalesce_params = ETHTOOL_COALESCE_USECS |
ETHTOOL_COALESCE_MAX_FRAMES, ETHTOOL_COALESCE_MAX_FRAMES |
ETHTOOL_COALESCE_USE_ADAPTIVE,
.supported_ring_params = ETHTOOL_RING_USE_RX_BUF_LEN | .supported_ring_params = ETHTOOL_RING_USE_RX_BUF_LEN |
ETHTOOL_RING_USE_CQE_SIZE, ETHTOOL_RING_USE_CQE_SIZE,
.get_link = otx2_get_link, .get_link = otx2_get_link,
......
...@@ -1254,6 +1254,7 @@ static irqreturn_t otx2_cq_intr_handler(int irq, void *cq_irq) ...@@ -1254,6 +1254,7 @@ static irqreturn_t otx2_cq_intr_handler(int irq, void *cq_irq)
otx2_write64(pf, NIX_LF_CINTX_ENA_W1C(qidx), BIT_ULL(0)); otx2_write64(pf, NIX_LF_CINTX_ENA_W1C(qidx), BIT_ULL(0));
/* Schedule NAPI */ /* Schedule NAPI */
pf->napi_events++;
napi_schedule_irqoff(&cq_poll->napi); napi_schedule_irqoff(&cq_poll->napi);
return IRQ_HANDLED; return IRQ_HANDLED;
...@@ -1267,6 +1268,7 @@ static void otx2_disable_napi(struct otx2_nic *pf) ...@@ -1267,6 +1268,7 @@ static void otx2_disable_napi(struct otx2_nic *pf)
for (qidx = 0; qidx < pf->hw.cint_cnt; qidx++) { for (qidx = 0; qidx < pf->hw.cint_cnt; qidx++) {
cq_poll = &qset->napi[qidx]; cq_poll = &qset->napi[qidx];
cancel_work_sync(&cq_poll->dim.work);
napi_disable(&cq_poll->napi); napi_disable(&cq_poll->napi);
netif_napi_del(&cq_poll->napi); netif_napi_del(&cq_poll->napi);
} }
...@@ -1546,6 +1548,24 @@ static void otx2_do_set_rx_mode(struct otx2_nic *pf) ...@@ -1546,6 +1548,24 @@ static void otx2_do_set_rx_mode(struct otx2_nic *pf)
mutex_unlock(&pf->mbox.lock); mutex_unlock(&pf->mbox.lock);
} }
static void otx2_dim_work(struct work_struct *w)
{
struct dim_cq_moder cur_moder;
struct otx2_cq_poll *cq_poll;
struct otx2_nic *pfvf;
struct dim *dim;
dim = container_of(w, struct dim, work);
cur_moder = net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
cq_poll = container_of(dim, struct otx2_cq_poll, dim);
pfvf = (struct otx2_nic *)cq_poll->dev;
pfvf->hw.cq_time_wait = (cur_moder.usec > CQ_TIMER_THRESH_MAX) ?
CQ_TIMER_THRESH_MAX : cur_moder.usec;
pfvf->hw.cq_ecount_wait = (cur_moder.pkts > NAPI_POLL_WEIGHT) ?
NAPI_POLL_WEIGHT : cur_moder.pkts;
dim->state = DIM_START_MEASURE;
}
int otx2_open(struct net_device *netdev) int otx2_open(struct net_device *netdev)
{ {
struct otx2_nic *pf = netdev_priv(netdev); struct otx2_nic *pf = netdev_priv(netdev);
...@@ -1612,6 +1632,8 @@ int otx2_open(struct net_device *netdev) ...@@ -1612,6 +1632,8 @@ int otx2_open(struct net_device *netdev)
cq_poll->cq_ids[CQ_XDP] = CINT_INVALID_CQ; cq_poll->cq_ids[CQ_XDP] = CINT_INVALID_CQ;
cq_poll->dev = (void *)pf; cq_poll->dev = (void *)pf;
cq_poll->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_CQE;
INIT_WORK(&cq_poll->dim.work, otx2_dim_work);
netif_napi_add(netdev, &cq_poll->napi, netif_napi_add(netdev, &cq_poll->napi,
otx2_napi_handler, NAPI_POLL_WEIGHT); otx2_napi_handler, NAPI_POLL_WEIGHT);
napi_enable(&cq_poll->napi); napi_enable(&cq_poll->napi);
......
...@@ -484,6 +484,18 @@ static int otx2_tx_napi_handler(struct otx2_nic *pfvf, ...@@ -484,6 +484,18 @@ static int otx2_tx_napi_handler(struct otx2_nic *pfvf,
return 0; return 0;
} }
static void otx2_adjust_adaptive_coalese(struct otx2_nic *pfvf, struct otx2_cq_poll *cq_poll)
{
struct dim_sample dim_sample;
u64 rx_frames, rx_bytes;
rx_frames = OTX2_GET_RX_STATS(RX_BCAST) + OTX2_GET_RX_STATS(RX_MCAST) +
OTX2_GET_RX_STATS(RX_UCAST);
rx_bytes = OTX2_GET_RX_STATS(RX_OCTS);
dim_update_sample(pfvf->napi_events, rx_frames, rx_bytes, &dim_sample);
net_dim(&cq_poll->dim, dim_sample);
}
int otx2_napi_handler(struct napi_struct *napi, int budget) int otx2_napi_handler(struct napi_struct *napi, int budget)
{ {
struct otx2_cq_queue *rx_cq = NULL; struct otx2_cq_queue *rx_cq = NULL;
...@@ -521,6 +533,17 @@ int otx2_napi_handler(struct napi_struct *napi, int budget) ...@@ -521,6 +533,17 @@ int otx2_napi_handler(struct napi_struct *napi, int budget)
if (pfvf->flags & OTX2_FLAG_INTF_DOWN) if (pfvf->flags & OTX2_FLAG_INTF_DOWN)
return workdone; return workdone;
/* Check for adaptive interrupt coalesce */
if (workdone != 0 &&
((pfvf->flags & OTX2_FLAG_ADPTV_INT_COAL_ENABLED) ==
OTX2_FLAG_ADPTV_INT_COAL_ENABLED)) {
/* Adjust irq coalese using net_dim */
otx2_adjust_adaptive_coalese(pfvf, cq_poll);
/* Update irq coalescing */
for (i = 0; i < pfvf->hw.cint_cnt; i++)
otx2_config_irq_coalescing(pfvf, i);
}
/* Re-enable interrupts */ /* Re-enable interrupts */
otx2_write64(pfvf, NIX_LF_CINTX_ENA_W1S(cq_poll->cint_idx), otx2_write64(pfvf, NIX_LF_CINTX_ENA_W1S(cq_poll->cint_idx),
BIT_ULL(0)); BIT_ULL(0));
......
...@@ -109,6 +109,7 @@ struct otx2_cq_poll { ...@@ -109,6 +109,7 @@ struct otx2_cq_poll {
#define CINT_INVALID_CQ 255 #define CINT_INVALID_CQ 255
u8 cint_idx; u8 cint_idx;
u8 cq_ids[CQS_PER_CINT]; u8 cq_ids[CQS_PER_CINT];
struct dim dim;
struct napi_struct napi; struct napi_struct napi;
}; };
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment