Minor tweaks and improvements

- Initialize server with Recv requests
- Reduce RX pool size to match other sizes
- Update access flags
- Pool CQ for completion of Recv requests

rdma_server.c

@@ -22,6 +22,7 @@ int main(void) {
 	struct udp_rdma_parameters local_params;
 	struct udp_rdma_parameters remote_params;
 
+	const char payload[32] = "Hello, world!";
 	void *memory_buffer;
 	int result;
 	int allocated_size = 16384 * sizeof(char);
@@ -55,7 +56,7 @@ int main(void) {
 
 	memset(memory_buffer, 0, allocated_size);
 
-	ibv_dev_mr = ibv_reg_mr(ibv_dev_pd, memory_buffer, allocated_size, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ);
+	ibv_dev_mr = ibv_reg_mr(ibv_dev_pd, memory_buffer, allocated_size, IBV_ACCESS_LOCAL_WRITE);
 	if(!ibv_dev_mr) {
 		fprintf(stderr, "Memory Region registration failed\n");
 		return -1;
@@ -64,7 +65,7 @@ int main(void) {
 	/******************************
 	 *** RDMA initialization (2) **
 	 ******************************/
-	ibv_dev_cq = ibv_create_cq(ibv_dev_ctx, 1025, NULL, NULL, 0);
+	ibv_dev_cq = ibv_create_cq(ibv_dev_ctx, 1, NULL, NULL, 0);
 	if(!ibv_dev_cq) {
 		fprintf(stderr, "Completion Queue creation failed\n");
 		return -1;
@@ -76,6 +77,11 @@ int main(void) {
 		return -1;
 	}
 
+	// RX buffer of size 1 is filled with Recv requests
+	if(!initialize_post_recv(ibv_dev_qp, ibv_dev_mr)) {
+		return -1;
+	}
+
 	/******************************
 	 **** Exchange informations ***
 	 ******************************/
@@ -109,11 +115,16 @@ int main(void) {
 	udp_rdma_dump_packet(&remote_params);
 
 	/******************************
-	 ******* RDMA connection ******
+	 ****** Await completion ******
 	 ******************************/
-	if(!set_peer_informations(ibv_dev_qp, remote_params.psn, remote_params.lid, remote_params.qpn, remote_params.gid)) {
-		return -1;
-	}
+	// Put payload in RDMA buffer
+	memcpy(memory_buffer, (void *) payload, sizeof(payload));
+
+	struct ibv_wc ibv_dev_wc;
+
+	do {
+		result = ibv_poll_cq(ibv_dev_cq, 1, &ibv_dev_wc);
+	} while(result == 0);
 
 	/******************************
 	 ******* Global cleanup *******

soft_rdma.c

@@ -51,7 +51,7 @@ struct ibv_qp *initialize_queue_pair(struct ibv_pd *ibv_dev_pd, struct ibv_cq *i
 	ibv_dev_qp_request.recv_cq = ibv_dev_cq;
 
 	ibv_dev_qp_request.cap.max_send_wr = 1;
-	ibv_dev_qp_request.cap.max_recv_wr = 1025;
+	ibv_dev_qp_request.cap.max_recv_wr = 1;
 	ibv_dev_qp_request.cap.max_send_sge = 1;
 	ibv_dev_qp_request.cap.max_recv_sge = 1;
 
@@ -61,11 +61,14 @@ struct ibv_qp *initialize_queue_pair(struct ibv_pd *ibv_dev_pd, struct ibv_cq *i
 		return NULL;
 	}
 
+	// XXX: added to try and make things work
+	ibv_query_qp(ibv_dev_qp, &ibv_dev_qp_params, IBV_QP_CAP, &ibv_dev_qp_request);
+
 	// Definition of QP state to "Initialized" and access control
 	ibv_dev_qp_params.qp_state = IBV_QPS_INIT;
 	ibv_dev_qp_params.pkey_index = 0;
 	ibv_dev_qp_params.port_num = 1;
-	ibv_dev_qp_params.qp_access_flags = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE;
+	ibv_dev_qp_params.qp_access_flags = 0;
 
 	result = ibv_modify_qp(ibv_dev_qp, &ibv_dev_qp_params, IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_ACCESS_FLAGS);
 
@@ -129,3 +132,31 @@ bool set_peer_informations(struct ibv_qp *ibv_dev_qp, uint32_t remote_psn, uint3
 	return true;
 }
 
+bool initialize_post_recv(struct ibv_qp *ibv_dev_qp, struct ibv_mr *ibv_dev_mr) {
+	struct ibv_sge ibv_dev_sge;
+	struct ibv_recv_wr ibv_dev_rdma_wr;
+	struct ibv_recv_wr *ibv_dev_bad_wr = NULL;
+
+	int result;
+
+	memset(&ibv_dev_sge, 0, sizeof(struct ibv_sge));
+	memset(&ibv_dev_rdma_wr, 0, sizeof(struct ibv_recv_wr));
+
+	ibv_dev_sge.addr = (uintptr_t) ibv_dev_mr->addr;
+	ibv_dev_sge.length = ibv_dev_mr->length;
+	ibv_dev_sge.lkey = ibv_dev_mr->lkey;
+
+	ibv_dev_rdma_wr.wr_id = (uint64_t) rand();
+	ibv_dev_rdma_wr.sg_list = &ibv_dev_sge;
+	ibv_dev_rdma_wr.num_sge = 1;
+
+	result = ibv_post_recv(ibv_dev_qp, &ibv_dev_rdma_wr, &ibv_dev_bad_wr);
+
+	if(result) {
+		perror("Could not post Recv request");
+		return false;
+	}
+
+	return ibv_dev_bad_wr == NULL;
+}
+

soft_rdma.h

 #include <stdbool.h>
 #include <stdio.h>
+#include <stdlib.h>
 
 #include <infiniband/verbs.h>
 
 struct ibv_context *initialize_device_context(const char *device_name);
 struct ibv_qp *initialize_queue_pair(struct ibv_pd *ibv_dev_pd, struct ibv_cq *ibv_dev_cq);
 bool set_peer_informations(struct ibv_qp *ibv_dev_qp, uint32_t remote_psn, uint32_t remote_lid, uint32_t remote_qpn, uint8_t *remote_gid);
+bool initialize_post_recv(struct ibv_qp *ibv_dev_qp, struct ibv_mr *ibv_dev_mr);