Commit 40442b30 authored by Mike Marciniszyn's avatar Mike Marciniszyn Committed by Jason Gunthorpe

IB/hfi1: Move rhf_offset from devdata to ctxtdata

This field should be in ctxtdata to allow for better locality of access by
eliminating a dd dereference.

The new field is now side-by-side with rcvhdrqentsize since the rhf_offset
is a function of the rcvhdrqentsize.

Both fields are now correctly sized as u8.
Reviewed-by: default avatarMichael J. Ruhl <michael.j.ruhl@intel.com>
Signed-off-by: default avatarMike Marciniszyn <mike.marciniszyn@intel.com>
Signed-off-by: default avatarDennis Dalessandro <dennis.dalessandro@intel.com>
Signed-off-by: default avatarJason Gunthorpe <jgg@mellanox.com>
parent b0ba3c18
...@@ -15217,11 +15217,6 @@ struct hfi1_devdata *hfi1_init_dd(struct pci_dev *pdev, ...@@ -15217,11 +15217,6 @@ struct hfi1_devdata *hfi1_init_dd(struct pci_dev *pdev,
aspm_init(dd); aspm_init(dd);
dd->rcvhdrsize = DEFAULT_RCVHDRSIZE; dd->rcvhdrsize = DEFAULT_RCVHDRSIZE;
/*
* rcd[0] is guaranteed to be valid by this point. Also, all
* context are using the same value, as per the module parameter.
*/
dd->rhf_offset = dd->rcd[0]->rcvhdrqentsize - sizeof(u64) / sizeof(u32);
ret = init_pervl_scs(dd); ret = init_pervl_scs(dd);
if (ret) if (ret)
......
...@@ -208,25 +208,25 @@ static inline void *get_egrbuf(const struct hfi1_ctxtdata *rcd, u64 rhf, ...@@ -208,25 +208,25 @@ static inline void *get_egrbuf(const struct hfi1_ctxtdata *rcd, u64 rhf,
(offset * RCV_BUF_BLOCK_SIZE)); (offset * RCV_BUF_BLOCK_SIZE));
} }
static inline void *hfi1_get_header(struct hfi1_devdata *dd, static inline void *hfi1_get_header(struct hfi1_ctxtdata *rcd,
__le32 *rhf_addr) __le32 *rhf_addr)
{ {
u32 offset = rhf_hdrq_offset(rhf_to_cpu(rhf_addr)); u32 offset = rhf_hdrq_offset(rhf_to_cpu(rhf_addr));
return (void *)(rhf_addr - dd->rhf_offset + offset); return (void *)(rhf_addr - rcd->rhf_offset + offset);
} }
static inline struct ib_header *hfi1_get_msgheader(struct hfi1_devdata *dd, static inline struct ib_header *hfi1_get_msgheader(struct hfi1_ctxtdata *rcd,
__le32 *rhf_addr) __le32 *rhf_addr)
{ {
return (struct ib_header *)hfi1_get_header(dd, rhf_addr); return (struct ib_header *)hfi1_get_header(rcd, rhf_addr);
} }
static inline struct hfi1_16b_header static inline struct hfi1_16b_header
*hfi1_get_16B_header(struct hfi1_devdata *dd, *hfi1_get_16B_header(struct hfi1_ctxtdata *rcd,
__le32 *rhf_addr) __le32 *rhf_addr)
{ {
return (struct hfi1_16b_header *)hfi1_get_header(dd, rhf_addr); return (struct hfi1_16b_header *)hfi1_get_header(rcd, rhf_addr);
} }
/* /*
...@@ -591,13 +591,12 @@ static void __prescan_rxq(struct hfi1_packet *packet) ...@@ -591,13 +591,12 @@ static void __prescan_rxq(struct hfi1_packet *packet)
init_ps_mdata(&mdata, packet); init_ps_mdata(&mdata, packet);
while (1) { while (1) {
struct hfi1_devdata *dd = rcd->dd;
struct hfi1_ibport *ibp = rcd_to_iport(rcd); struct hfi1_ibport *ibp = rcd_to_iport(rcd);
__le32 *rhf_addr = (__le32 *)rcd->rcvhdrq + mdata.ps_head + __le32 *rhf_addr = (__le32 *)rcd->rcvhdrq + mdata.ps_head +
dd->rhf_offset; packet->rcd->rhf_offset;
struct rvt_qp *qp; struct rvt_qp *qp;
struct ib_header *hdr; struct ib_header *hdr;
struct rvt_dev_info *rdi = &dd->verbs_dev.rdi; struct rvt_dev_info *rdi = &rcd->dd->verbs_dev.rdi;
u64 rhf = rhf_to_cpu(rhf_addr); u64 rhf = rhf_to_cpu(rhf_addr);
u32 etype = rhf_rcv_type(rhf), qpn, bth1; u32 etype = rhf_rcv_type(rhf), qpn, bth1;
int is_ecn = 0; int is_ecn = 0;
...@@ -612,7 +611,7 @@ static void __prescan_rxq(struct hfi1_packet *packet) ...@@ -612,7 +611,7 @@ static void __prescan_rxq(struct hfi1_packet *packet)
if (etype != RHF_RCV_TYPE_IB) if (etype != RHF_RCV_TYPE_IB)
goto next; goto next;
packet->hdr = hfi1_get_msgheader(dd, rhf_addr); packet->hdr = hfi1_get_msgheader(packet->rcd, rhf_addr);
hdr = packet->hdr; hdr = packet->hdr;
lnh = ib_get_lnh(hdr); lnh = ib_get_lnh(hdr);
...@@ -718,7 +717,7 @@ static noinline int skip_rcv_packet(struct hfi1_packet *packet, int thread) ...@@ -718,7 +717,7 @@ static noinline int skip_rcv_packet(struct hfi1_packet *packet, int thread)
ret = check_max_packet(packet, thread); ret = check_max_packet(packet, thread);
packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff + packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff +
packet->rcd->dd->rhf_offset; packet->rcd->rhf_offset;
packet->rhf = rhf_to_cpu(packet->rhf_addr); packet->rhf = rhf_to_cpu(packet->rhf_addr);
return ret; return ret;
...@@ -768,7 +767,7 @@ static inline int process_rcv_packet(struct hfi1_packet *packet, int thread) ...@@ -768,7 +767,7 @@ static inline int process_rcv_packet(struct hfi1_packet *packet, int thread)
ret = check_max_packet(packet, thread); ret = check_max_packet(packet, thread);
packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff + packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff +
packet->rcd->dd->rhf_offset; packet->rcd->rhf_offset;
packet->rhf = rhf_to_cpu(packet->rhf_addr); packet->rhf = rhf_to_cpu(packet->rhf_addr);
return ret; return ret;
...@@ -949,12 +948,12 @@ static inline int set_armed_to_active(struct hfi1_ctxtdata *rcd, ...@@ -949,12 +948,12 @@ static inline int set_armed_to_active(struct hfi1_ctxtdata *rcd,
u8 sc = SC15_PACKET; u8 sc = SC15_PACKET;
if (etype == RHF_RCV_TYPE_IB) { if (etype == RHF_RCV_TYPE_IB) {
struct ib_header *hdr = hfi1_get_msgheader(packet->rcd->dd, struct ib_header *hdr = hfi1_get_msgheader(packet->rcd,
packet->rhf_addr); packet->rhf_addr);
sc = hfi1_9B_get_sc5(hdr, packet->rhf); sc = hfi1_9B_get_sc5(hdr, packet->rhf);
} else if (etype == RHF_RCV_TYPE_BYPASS) { } else if (etype == RHF_RCV_TYPE_BYPASS) {
struct hfi1_16b_header *hdr = hfi1_get_16B_header( struct hfi1_16b_header *hdr = hfi1_get_16B_header(
packet->rcd->dd, packet->rcd,
packet->rhf_addr); packet->rhf_addr);
sc = hfi1_16B_get_sc(hdr); sc = hfi1_16B_get_sc(hdr);
} }
...@@ -1034,7 +1033,7 @@ int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread) ...@@ -1034,7 +1033,7 @@ int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread)
packet.rhqoff += packet.rsize; packet.rhqoff += packet.rsize;
packet.rhf_addr = (__le32 *)rcd->rcvhdrq + packet.rhf_addr = (__le32 *)rcd->rcvhdrq +
packet.rhqoff + packet.rhqoff +
dd->rhf_offset; rcd->rhf_offset;
packet.rhf = rhf_to_cpu(packet.rhf_addr); packet.rhf = rhf_to_cpu(packet.rhf_addr);
} else if (skip_pkt) { } else if (skip_pkt) {
...@@ -1384,7 +1383,7 @@ int hfi1_reset_device(int unit) ...@@ -1384,7 +1383,7 @@ int hfi1_reset_device(int unit)
static inline void hfi1_setup_ib_header(struct hfi1_packet *packet) static inline void hfi1_setup_ib_header(struct hfi1_packet *packet)
{ {
packet->hdr = (struct hfi1_ib_message_header *) packet->hdr = (struct hfi1_ib_message_header *)
hfi1_get_msgheader(packet->rcd->dd, hfi1_get_msgheader(packet->rcd,
packet->rhf_addr); packet->rhf_addr);
packet->hlen = (u8 *)packet->rhf_addr - (u8 *)packet->hdr; packet->hlen = (u8 *)packet->rhf_addr - (u8 *)packet->hdr;
} }
...@@ -1485,7 +1484,7 @@ static int hfi1_setup_bypass_packet(struct hfi1_packet *packet) ...@@ -1485,7 +1484,7 @@ static int hfi1_setup_bypass_packet(struct hfi1_packet *packet)
u8 l4; u8 l4;
packet->hdr = (struct hfi1_16b_header *) packet->hdr = (struct hfi1_16b_header *)
hfi1_get_16B_header(packet->rcd->dd, hfi1_get_16B_header(packet->rcd,
packet->rhf_addr); packet->rhf_addr);
l4 = hfi1_16B_get_l4(packet->hdr); l4 = hfi1_16B_get_l4(packet->hdr);
if (l4 == OPA_16B_L4_IB_LOCAL) { if (l4 == OPA_16B_L4_IB_LOCAL) {
...@@ -1719,9 +1718,8 @@ void seqfile_dump_rcd(struct seq_file *s, struct hfi1_ctxtdata *rcd) ...@@ -1719,9 +1718,8 @@ void seqfile_dump_rcd(struct seq_file *s, struct hfi1_ctxtdata *rcd)
init_ps_mdata(&mdata, &packet); init_ps_mdata(&mdata, &packet);
while (1) { while (1) {
struct hfi1_devdata *dd = rcd->dd;
__le32 *rhf_addr = (__le32 *)rcd->rcvhdrq + mdata.ps_head + __le32 *rhf_addr = (__le32 *)rcd->rcvhdrq + mdata.ps_head +
dd->rhf_offset; rcd->rhf_offset;
struct ib_header *hdr; struct ib_header *hdr;
u64 rhf = rhf_to_cpu(rhf_addr); u64 rhf = rhf_to_cpu(rhf_addr);
u32 etype = rhf_rcv_type(rhf), qpn; u32 etype = rhf_rcv_type(rhf), qpn;
...@@ -1738,7 +1736,7 @@ void seqfile_dump_rcd(struct seq_file *s, struct hfi1_ctxtdata *rcd) ...@@ -1738,7 +1736,7 @@ void seqfile_dump_rcd(struct seq_file *s, struct hfi1_ctxtdata *rcd)
if (etype > RHF_RCV_TYPE_IB) if (etype > RHF_RCV_TYPE_IB)
goto next; goto next;
packet.hdr = hfi1_get_msgheader(dd, rhf_addr); packet.hdr = hfi1_get_msgheader(rcd, rhf_addr);
hdr = packet.hdr; hdr = packet.hdr;
lnh = be16_to_cpu(hdr->lrh[0]) & 3; lnh = be16_to_cpu(hdr->lrh[0]) & 3;
......
...@@ -206,7 +206,9 @@ struct hfi1_ctxtdata { ...@@ -206,7 +206,9 @@ struct hfi1_ctxtdata {
/* number of rcvhdrq entries */ /* number of rcvhdrq entries */
u16 rcvhdrq_cnt; u16 rcvhdrq_cnt;
/* size of each of the rcvhdrq entries */ /* size of each of the rcvhdrq entries */
u16 rcvhdrqentsize; u8 rcvhdrqentsize;
/* offset of RHF within receive header entry */
u8 rhf_offset;
/* mmap of hdrq, must fit in 44 bits */ /* mmap of hdrq, must fit in 44 bits */
dma_addr_t rcvhdrq_dma; dma_addr_t rcvhdrq_dma;
dma_addr_t rcvhdrqtailaddr_dma; dma_addr_t rcvhdrqtailaddr_dma;
...@@ -1330,7 +1332,6 @@ struct hfi1_devdata { ...@@ -1330,7 +1332,6 @@ struct hfi1_devdata {
seqlock_t sc2vl_lock ____cacheline_aligned_in_smp; seqlock_t sc2vl_lock ____cacheline_aligned_in_smp;
u64 sc2vl[4]; u64 sc2vl[4];
u64 __percpu *rcv_limit; u64 __percpu *rcv_limit;
u16 rhf_offset; /* offset of RHF within receive header entry */
/* adding a new field here would make it part of this cacheline */ /* adding a new field here would make it part of this cacheline */
/* OUI comes from the HW. Used everywhere as 3 separate bytes. */ /* OUI comes from the HW. Used everywhere as 3 separate bytes. */
...@@ -1469,7 +1470,7 @@ void hfi1_make_ud_req_16B(struct rvt_qp *qp, ...@@ -1469,7 +1470,7 @@ void hfi1_make_ud_req_16B(struct rvt_qp *qp,
/* calculate the current RHF address */ /* calculate the current RHF address */
static inline __le32 *get_rhf_addr(struct hfi1_ctxtdata *rcd) static inline __le32 *get_rhf_addr(struct hfi1_ctxtdata *rcd)
{ {
return (__le32 *)rcd->rcvhdrq + rcd->head + rcd->dd->rhf_offset; return (__le32 *)rcd->rcvhdrq + rcd->head + rcd->rhf_offset;
} }
int hfi1_reset_device(int); int hfi1_reset_device(int);
......
...@@ -405,6 +405,8 @@ int hfi1_create_ctxtdata(struct hfi1_pportdata *ppd, int numa, ...@@ -405,6 +405,8 @@ int hfi1_create_ctxtdata(struct hfi1_pportdata *ppd, int numa,
rcd->rcvhdrq_cnt = rcvhdrcnt; rcd->rcvhdrq_cnt = rcvhdrcnt;
rcd->rcvhdrqentsize = hfi1_hdrq_entsize; rcd->rcvhdrqentsize = hfi1_hdrq_entsize;
rcd->rhf_offset =
rcd->rcvhdrqentsize - sizeof(u64) / sizeof(u32);
/* /*
* Simple Eager buffer allocation: we have already pre-allocated * Simple Eager buffer allocation: we have already pre-allocated
* the number of RcvArray entry groups. Each ctxtdata structure * the number of RcvArray entry groups. Each ctxtdata structure
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment