struct ibv_mr *start_mr;\r
\r
enum test_state state; /* used for cond/signalling */\r
- sem_t sem;\r
+// sem_t sem;\r
\r
struct sockaddr_in sin;\r
uint16_t port; /* dst port in NBO */\r
int validate; /* validate ping data */\r
\r
/* CM stuff */\r
- pthread_t cmthread;\r
+// pthread_t cmthread;\r
struct rdma_event_channel *cm_channel;\r
struct rdma_cm_id *cm_id; /* connection on client side,*/\r
/* listener on service side. */\r
struct rdma_cm_id *child_cm_id; /* connection on server side */\r
};\r
\r
+struct rping_cb *cb;\r
+static void *cm_thread(void *arg);\r
+static void *cq_thread(void *arg);\r
+\r
static int rping_cma_event_handler(struct rdma_cm_id *cma_id,\r
struct rdma_cm_event *event)\r
{\r
if (ret) {\r
cb->state = RDMA_ERROR;\r
fprintf(stderr, "rdma_resolve_route error %d\n", ret);\r
- sem_post(&cb->sem);\r
+// sem_post(&cb->sem);\r
}\r
break;\r
\r
case RDMA_CM_EVENT_ROUTE_RESOLVED:\r
cb->state = ROUTE_RESOLVED;\r
- sem_post(&cb->sem);\r
+// sem_post(&cb->sem);\r
break;\r
\r
case RDMA_CM_EVENT_CONNECT_REQUEST:\r
cb->state = CONNECT_REQUEST;\r
cb->child_cm_id = cma_id;\r
DEBUG_LOG("child cma %p\n", cb->child_cm_id);\r
- sem_post(&cb->sem);\r
+// sem_post(&cb->sem);\r
break;\r
\r
case RDMA_CM_EVENT_ESTABLISHED:\r
if (!cb->server) {\r
cb->state = CONNECTED;\r
}\r
- sem_post(&cb->sem);\r
+// sem_post(&cb->sem);\r
break;\r
\r
case RDMA_CM_EVENT_ADDR_ERROR:\r
case RDMA_CM_EVENT_REJECTED:\r
fprintf(stderr, "cma event %s, error %d\n",\r
rdma_event_str(event->event), event->status);\r
- sem_post(&cb->sem);\r
+// sem_post(&cb->sem);\r
ret = -1;\r
break;\r
\r
case RDMA_CM_EVENT_DISCONNECTED:\r
fprintf(stderr, "%s DISCONNECT EVENT...\n",\r
cb->server ? "server" : "client");\r
- sem_post(&cb->sem);\r
+// sem_post(&cb->sem);\r
break;\r
\r
case RDMA_CM_EVENT_DEVICE_REMOVAL:\r
case IBV_WC_RDMA_WRITE:\r
DEBUG_LOG("rdma write completion\n");\r
cb->state = RDMA_WRITE_COMPLETE;\r
- sem_post(&cb->sem);\r
+// sem_post(&cb->sem);\r
break;\r
\r
case IBV_WC_RDMA_READ:\r
DEBUG_LOG("rdma read completion\n");\r
cb->state = RDMA_READ_COMPLETE;\r
- sem_post(&cb->sem);\r
+// sem_post(&cb->sem);\r
break;\r
\r
case IBV_WC_RECV:\r
fprintf(stderr, "post recv error: %d\n", ret);\r
goto error;\r
}\r
- sem_post(&cb->sem);\r
+// sem_post(&cb->sem);\r
break;\r
\r
default:\r
\r
error:\r
cb->state = RDMA_ERROR;\r
- sem_post(&cb->sem);\r
+// sem_post(&cb->sem);\r
return ret;\r
}\r
\r
return ret;\r
}\r
\r
- sem_wait(&cb->sem);\r
+ cm_thread(cb);\r
+// sem_wait(&cb->sem);\r
if (cb->state == RDMA_ERROR) {\r
fprintf(stderr, "wait for CONNECTED state %d\n", cb->state);\r
return -1;\r
return ret;\r
}\r
\r
-static void *cm_thread(void *arg)\r
+static void cm_thread(void *arg)\r
{\r
struct rping_cb *cb = arg;\r
struct rdma_cm_event *event;\r
int ret;\r
\r
- while (1) {\r
- ret = rdma_get_cm_event(cb->cm_channel, &event);\r
- if (ret) {\r
- fprintf(stderr, "rdma_get_cm_event err %d\n", ret);\r
- exit(ret);\r
- }\r
- ret = rping_cma_event_handler(event->id, event);\r
- rdma_ack_cm_event(event);\r
- if (ret)\r
- exit(ret);\r
+// while (1) {\r
+ ret = rdma_get_cm_event(cb->cm_channel, &event);\r
+ if (ret) {\r
+ fprintf(stderr, "rdma_get_cm_event err %d\n", ret);\r
+ return;\r
+// exit(ret);\r
}\r
+ ret = rping_cma_event_handler(event->id, event);\r
+ rdma_ack_cm_event(event);\r
+// if (ret)\r
+// exit(ret);\r
+// }\r
}\r
\r
-static void *cq_thread(void *arg)\r
+static void cq_thread(void *arg)\r
{\r
struct rping_cb *cb = arg;\r
struct ibv_cq *ev_cq;\r
void *ev_ctx;\r
int ret;\r
\r
- DEBUG_LOG("cq_thread started.\n");\r
+// DEBUG_LOG("cq_thread started.\n");\r
\r
- while (1) { \r
- pthread_testcancel();\r
+// while (1) { \r
+// pthread_testcancel();\r
\r
- ret = ibv_get_cq_event(cb->channel, &ev_cq, &ev_ctx);\r
- if (ret) {\r
- fprintf(stderr, "Failed to get cq event!\n");\r
- pthread_exit(NULL);\r
- }\r
- if (ev_cq != cb->cq) {\r
- fprintf(stderr, "Unknown CQ!\n");\r
- pthread_exit(NULL);\r
- }\r
- ret = ibv_req_notify_cq(cb->cq, 0);\r
- if (ret) {\r
- fprintf(stderr, "Failed to set notify!\n");\r
- pthread_exit(NULL);\r
- }\r
- ret = rping_cq_event_handler(cb);\r
- ibv_ack_cq_events(cb->cq, 1);\r
- if (ret)\r
- pthread_exit(NULL);\r
+ ret = ibv_get_cq_event(cb->channel, &ev_cq, &ev_ctx);\r
+ if (ret) {\r
+ fprintf(stderr, "Failed to get cq event!\n");\r
+ return;\r
+// pthread_exit(NULL);\r
+ }\r
+ if (ev_cq != cb->cq) {\r
+ fprintf(stderr, "Unknown CQ!\n");\r
+ return;\r
+// pthread_exit(NULL);\r
+ }\r
+ ret = ibv_req_notify_cq(cb->cq, 0);\r
+ if (ret) {\r
+ fprintf(stderr, "Failed to set notify!\n");\r
+ return;\r
+// pthread_exit(NULL);\r
}\r
+ ret = rping_cq_event_handler(cb);\r
+ ibv_ack_cq_events(cb->cq, 1);\r
+// pthread_exit(NULL);\r
+// }\r
}\r
\r
static void rping_format_send(struct rping_cb *cb, char *buf, struct ibv_mr *mr)\r
\r
while (1) {\r
/* Wait for client's Start STAG/TO/Len */\r
- sem_wait(&cb->sem);\r
+ cq_thread(cb);\r
+// sem_wait(&cb->sem);\r
if (cb->state != RDMA_READ_ADV) {\r
fprintf(stderr, "wait for RDMA_READ_ADV state %d\n",\r
cb->state);\r
DEBUG_LOG("server posted rdma read req \n");\r
\r
/* Wait for read completion */\r
- sem_wait(&cb->sem);\r
+ cq_thread(cb);\r
+// sem_wait(&cb->sem);\r
if (cb->state != RDMA_READ_COMPLETE) {\r
fprintf(stderr, "wait for RDMA_READ_COMPLETE state %d\n",\r
cb->state);\r
DEBUG_LOG("server posted go ahead\n");\r
\r
/* Wait for client's RDMA STAG/TO/Len */\r
- sem_wait(&cb->sem);\r
+ cq_thread(cb);\r
+// sem_wait(&cb->sem);\r
if (cb->state != RDMA_WRITE_ADV) {\r
fprintf(stderr, "wait for RDMA_WRITE_ADV state %d\n",\r
cb->state);\r
}\r
\r
/* Wait for completion */\r
- ret = sem_wait(&cb->sem);\r
+ cq_thread(cb);\r
+// ret = sem_wait(&cb->sem);\r
if (cb->state != RDMA_WRITE_COMPLETE) {\r
fprintf(stderr, "wait for RDMA_WRITE_COMPLETE state %d\n",\r
cb->state);\r
if (ret)\r
return ret;\r
\r
- sem_wait(&cb->sem);\r
+ cm_thread(cb);\r
+// sem_wait(&cb->sem);\r
if (cb->state != CONNECT_REQUEST) {\r
fprintf(stderr, "wait for CONNECT_REQUEST state %d\n",\r
cb->state);\r
goto err2;\r
}\r
\r
- pthread_create(&cb->cqthread, NULL, cq_thread, cb);\r
+// pthread_create(&cb->cqthread, NULL, cq_thread, cb);\r
\r
ret = rping_accept(cb);\r
if (ret) {\r
}\r
\r
/* Wait for server to ACK */\r
- sem_wait(&cb->sem);\r
+ cq_thread(cb);\r
+// sem_wait(&cb->sem);\r
if (cb->state != RDMA_WRITE_ADV) {\r
fprintf(stderr, "wait for RDMA_WRITE_ADV state %d\n",\r
cb->state);\r
}\r
\r
/* Wait for the server to say the RDMA Write is complete. */\r
- sem_wait(&cb->sem);\r
+ cq_thread(cb);\r
+// sem_wait(&cb->sem);\r
if (cb->state != RDMA_WRITE_COMPLETE) {\r
fprintf(stderr, "wait for RDMA_WRITE_COMPLETE state %d\n",\r
cb->state);\r
return ret;\r
}\r
\r
- sem_wait(&cb->sem);\r
+ cm_thread(cb);\r
+// sem_wait(&cb->sem);\r
if (cb->state != CONNECTED) {\r
fprintf(stderr, "wait for CONNECTED state %d\n", cb->state);\r
return -1;\r
return ret;\r
}\r
\r
- sem_wait(&cb->sem);\r
+ cm_thread(cb);\r
+// sem_wait(&cb->sem);\r
if (cb->state != ROUTE_RESOLVED) {\r
fprintf(stderr, "waiting for addr/route resolution state %d\n",\r
cb->state);\r
goto err2;\r
}\r
\r
- pthread_create(&cb->cqthread, NULL, cq_thread, cb);\r
+// pthread_create(&cb->cqthread, NULL, cq_thread, cb);\r
\r
ret = rping_connect_client(cb);\r
if (ret) {\r
\r
int main(int argc, char *argv[])\r
{\r
- struct rping_cb *cb;\r
+// struct rping_cb *cb;\r
int op;\r
int ret = 0;\r
\r
cb->size = 64;\r
cb->sin.sin_family = PF_INET;\r
cb->port = htons(7174);\r
- sem_init(&cb->sem, 0, 0);\r
+// sem_init(&cb->sem, 0, 0);\r
\r
opterr = 0;\r
while ((op=getopt(argc, argv, "a:Pp:C:S:t:scvVd")) != -1) {\r
}\r
DEBUG_LOG("created cm_id %p\n", cb->cm_id);\r
\r
- pthread_create(&cb->cmthread, NULL, cm_thread, cb);\r
+// pthread_create(&cb->cmthread, NULL, cm_thread, cb);\r
\r
if (cb->server)\r
ret = rping_run_server(cb);\r