From 62c2bddeaf5275425e1a7e3add59c3913ccdb4e9 Mon Sep 17 00:00:00 2001 From: Sean Hefty Date: Wed, 14 Jan 2009 14:31:34 -0800 Subject: [PATCH] cmatose: avoid missing completions cmatose uses a single CQ for send and receive completions. It then counts completions to determine if all sends and receives are done. It's possible for a receive completion to be polled when the intent is to count send completions. (See server side polling for sends done, then receives done. The poll will get up to 8 completions, which can lead to sends and receives being polled together.) Fix this by separating the send and receive completions to their own CQs to avoid any issues knowing what type of receives have been polled from the CQ. Signed-off-by: Sean Hefty --- examples/cmatose.c | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/examples/cmatose.c b/examples/cmatose.c index ba6299eb..75bc478f 100644 --- a/examples/cmatose.c +++ b/examples/cmatose.c @@ -58,11 +58,16 @@ struct cmatest_node { struct rdma_cm_id *cma_id; int connected; struct ibv_pd *pd; - struct ibv_cq *cq; + struct ibv_cq *cq[2]; struct ibv_mr *mr; void *mem; }; +enum CQ_INDEX { + SEND_CQ_INDEX, + RECV_CQ_INDEX +}; + struct cmatest { struct rdma_event_channel *channel; struct cmatest_node *nodes; @@ -124,24 +129,25 @@ static int init_node(struct cmatest_node *node) goto out; } - cqe = message_count ? message_count * 2 : 2; - node->cq = ibv_create_cq(node->cma_id->verbs, cqe, node, 0, 0); - if (!node->cq) { + cqe = message_count ? message_count : 1; + node->cq[SEND_CQ_INDEX] = ibv_create_cq(node->cma_id->verbs, cqe, node, 0, 0); + node->cq[RECV_CQ_INDEX] = ibv_create_cq(node->cma_id->verbs, cqe, node, 0, 0); + if (!node->cq[SEND_CQ_INDEX] || !node->cq[RECV_CQ_INDEX]) { ret = -ENOMEM; printf("cmatose: unable to create CQ\n"); goto out; } memset(&init_qp_attr, 0, sizeof init_qp_attr); - init_qp_attr.cap.max_send_wr = message_count ? message_count : 1; - init_qp_attr.cap.max_recv_wr = message_count ? message_count : 1; + init_qp_attr.cap.max_send_wr = cqe; + init_qp_attr.cap.max_recv_wr = cqe; init_qp_attr.cap.max_send_sge = 1; init_qp_attr.cap.max_recv_sge = 1; init_qp_attr.qp_context = node; init_qp_attr.sq_sig_all = 1; init_qp_attr.qp_type = IBV_QPT_RC; - init_qp_attr.send_cq = node->cq; - init_qp_attr.recv_cq = node->cq; + init_qp_attr.send_cq = node->cq[SEND_CQ_INDEX]; + init_qp_attr.recv_cq = node->cq[RECV_CQ_INDEX]; ret = rdma_create_qp(node->cma_id, node->pd, &init_qp_attr); if (ret) { printf("cmatose: unable to create QP: %d\n", ret); @@ -356,8 +362,11 @@ static void destroy_node(struct cmatest_node *node) if (node->cma_id->qp) rdma_destroy_qp(node->cma_id); - if (node->cq) - ibv_destroy_cq(node->cq); + if (node->cq[SEND_CQ_INDEX]) + ibv_destroy_cq(node->cq[SEND_CQ_INDEX]); + + if (node->cq[RECV_CQ_INDEX]) + ibv_destroy_cq(node->cq[RECV_CQ_INDEX]); if (node->mem) { ibv_dereg_mr(node->mr); @@ -409,7 +418,7 @@ static void destroy_nodes(void) free(test.nodes); } -static int poll_cqs(void) +static int poll_cqs(enum CQ_INDEX index) { struct ibv_wc wc[8]; int done, i, ret; @@ -419,7 +428,7 @@ static int poll_cqs(void) continue; for (done = 0; done < message_count; done += ret) { - ret = ibv_poll_cq(test.nodes[i].cq, 8, wc); + ret = ibv_poll_cq(test.nodes[i].cq[index], 8, wc); if (ret < 0) { printf("cmatose: failed polling CQ: %d\n", ret); return ret; @@ -563,12 +572,12 @@ static int run_server(void) } printf("completing sends\n"); - ret = poll_cqs(); + ret = poll_cqs(SEND_CQ_INDEX); if (ret) goto out; printf("receiving data transfers\n"); - ret = poll_cqs(); + ret = poll_cqs(RECV_CQ_INDEX); if (ret) goto out; printf("data transfers complete\n"); @@ -634,7 +643,7 @@ static int run_client(void) if (message_count) { printf("receiving data transfers\n"); - ret = poll_cqs(); + ret = poll_cqs(RECV_CQ_INDEX); if (ret) goto disc; -- 2.46.0