bnxt_en: Modify the NAPI logic for the new P7 chips

Modify the NAPI logic for the new doorbell mechanism on P7 chips.
These changes are compatible with the current P5 chips.

In the current logic, bnxt_poll_p5() services 1 or more CQs for each
MSIX.  Each MSIX has an associated NQ and each NQ has 1 or more
associated CQs.  If any CQ reaches NAPI budget, we'll stay in polling
mode and will unconditionally check and service all CQs until we exit
polling.  We always re-arm all CQs when we exit polling.

To be compatible with the new Toggle bit mechanism in P7 chips, we need
to modify the logic so that we service and re-arm the CQ only if we
receive an NQE notification for work for that CQ.  We add a new
had_nqe_notify bit to the cp_ring_info structure and it gets set when we
see the NQE notification for that CQ anytime during polling.  We'll
service and re-arm only the CQs with the had_nqe_notify bits set.
Reviewed-by: Somnath Kotur <somnath.kotur@broadcom.com>
Reviewed-by: Pavan Chebbi <pavan.chebbi@broadcom.com>
Signed-off-by: Michael Chan <michael.chan@broadcom.com>
Link: https://lore.kernel.org/r/20231120234405.194542-13-michael.chan@broadcom.comSigned-off-by: Jakub Kicinski <kuba@kernel.org>

drivers/net/ethernet/broadcom/bnxt/bnxt.c

@@ -2854,9 +2854,12 @@ static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
 	for (i = 0; i < cpr->cp_ring_count; i++) {
 		struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[i];
 
-		work_done += __bnxt_poll_work(bp, cpr2, budget - work_done);
+		if (cpr2->had_nqe_notify) {
+			work_done += __bnxt_poll_work(bp, cpr2,
+						      budget - work_done);
 			cpr->has_more_work |= cpr2->has_more_work;
 		}
+	}
 	return work_done;
 }
 
@@ -2876,6 +2879,8 @@ static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi,
 				    DB_RING_IDX(db, cpr2->cp_raw_cons),
 				    db->doorbell);
 			cpr2->had_work_done = 0;
+			if (dbr_type == DBR_TYPE_CQ_ARMALL)
+				cpr2->had_nqe_notify = 0;
 		}
 	}
 	__bnxt_poll_work_done(bp, bnapi, budget);
@@ -2934,6 +2939,7 @@ static int bnxt_poll_p5(struct napi_struct *napi, int budget)
 
 			idx = BNXT_NQ_HDL_IDX(idx);
 			cpr2 = &cpr->cp_ring_arr[idx];
+			cpr2->had_nqe_notify = 1;
 			work_done += __bnxt_poll_work(bp, cpr2,
 						      budget - work_done);
 			cpr->has_more_work |= cpr2->has_more_work;

drivers/net/ethernet/broadcom/bnxt/bnxt.h

@@ -1017,6 +1017,8 @@ struct bnxt_cp_ring_info {
 
 	u8			had_work_done:1;
 	u8			has_more_work:1;
+	u8			had_nqe_notify:1;
+
 	u8			cp_ring_type;
 	u8			cp_idx;