rping snapshot of work in progress.

Signed-off-by: Sean Hefty <sean.hefty@intel.com>
git-svn-id: svn://openib.tc.cornell.edu/gen1@1849 ad392aa1-c5ef-ae45-8dd8-e69d62a5ef86

diff --git a/branches/winverbs/ulp/librdmacm/examples/rping/rping.c b/branches/winverbs/ulp/librdmacm/examples/rping/rping.c
index f0d757275f1dc7d2f30c768374dd47faf3b2c9a3..1e264feb2487d6b450d61c468589852584931b7c 100644 (file)
--- a/branches/winverbs/ulp/librdmacm/examples/rping/rping.c
+++ b/branches/winverbs/ulp/librdmacm/examples/rping/rping.c
@@ -122,7 +122,7 @@ struct rping_cb {
        struct ibv_mr *start_mr;\r
 \r
        enum test_state state;          /* used for cond/signalling */\r
-       sem_t sem;\r
+//     sem_t sem;\r
 \r
        struct sockaddr_in sin;\r
        uint16_t port;                  /* dst port in NBO */\r
@@ -132,13 +132,17 @@ struct rping_cb {
        int validate;                   /* validate ping data */\r
 \r
        /* CM stuff */\r
-       pthread_t cmthread;\r
+//     pthread_t cmthread;\r
        struct rdma_event_channel *cm_channel;\r
        struct rdma_cm_id *cm_id;       /* connection on client side,*/\r
                                        /* listener on service side. */\r
        struct rdma_cm_id *child_cm_id; /* connection on server side */\r
 };\r
 \r
+struct rping_cb *cb;\r
+static void *cm_thread(void *arg);\r
+static void *cq_thread(void *arg);\r
+\r
 static int rping_cma_event_handler(struct rdma_cm_id *cma_id,\r
                                    struct rdma_cm_event *event)\r
 {\r
@@ -156,20 +160,20 @@ static int rping_cma_event_handler(struct rdma_cm_id *cma_id,
                if (ret) {\r
                        cb->state = RDMA_ERROR;\r
                        fprintf(stderr, "rdma_resolve_route error %d\n", ret);\r
-                       sem_post(&cb->sem);\r
+//                     sem_post(&cb->sem);\r
                }\r
                break;\r
 \r
        case RDMA_CM_EVENT_ROUTE_RESOLVED:\r
                cb->state = ROUTE_RESOLVED;\r
-               sem_post(&cb->sem);\r
+//             sem_post(&cb->sem);\r
                break;\r
 \r
        case RDMA_CM_EVENT_CONNECT_REQUEST:\r
                cb->state = CONNECT_REQUEST;\r
                cb->child_cm_id = cma_id;\r
                DEBUG_LOG("child cma %p\n", cb->child_cm_id);\r
-               sem_post(&cb->sem);\r
+//             sem_post(&cb->sem);\r
                break;\r
 \r
        case RDMA_CM_EVENT_ESTABLISHED:\r
@@ -181,7 +185,7 @@ static int rping_cma_event_handler(struct rdma_cm_id *cma_id,
                if (!cb->server) {\r
                        cb->state = CONNECTED;\r
                }\r
-               sem_post(&cb->sem);\r
+//             sem_post(&cb->sem);\r
                break;\r
 \r
        case RDMA_CM_EVENT_ADDR_ERROR:\r
@@ -191,14 +195,14 @@ static int rping_cma_event_handler(struct rdma_cm_id *cma_id,
        case RDMA_CM_EVENT_REJECTED:\r
                fprintf(stderr, "cma event %s, error %d\n",\r
                        rdma_event_str(event->event), event->status);\r
-               sem_post(&cb->sem);\r
+//             sem_post(&cb->sem);\r
                ret = -1;\r
                break;\r
 \r
        case RDMA_CM_EVENT_DISCONNECTED:\r
                fprintf(stderr, "%s DISCONNECT EVENT...\n",\r
                        cb->server ? "server" : "client");\r
-               sem_post(&cb->sem);\r
+//             sem_post(&cb->sem);\r
                break;\r
 \r
        case RDMA_CM_EVENT_DEVICE_REMOVAL:\r
@@ -276,13 +280,13 @@ static int rping_cq_event_handler(struct rping_cb *cb)
                case IBV_WC_RDMA_WRITE:\r
                        DEBUG_LOG("rdma write completion\n");\r
                        cb->state = RDMA_WRITE_COMPLETE;\r
-                       sem_post(&cb->sem);\r
+//                     sem_post(&cb->sem);\r
                        break;\r
 \r
                case IBV_WC_RDMA_READ:\r
                        DEBUG_LOG("rdma read completion\n");\r
                        cb->state = RDMA_READ_COMPLETE;\r
-                       sem_post(&cb->sem);\r
+//                     sem_post(&cb->sem);\r
                        break;\r
 \r
                case IBV_WC_RECV:\r
@@ -299,7 +303,7 @@ static int rping_cq_event_handler(struct rping_cb *cb)
                                fprintf(stderr, "post recv error: %d\n", ret);\r
                                goto error;\r
                        }\r
-                       sem_post(&cb->sem);\r
+//                     sem_post(&cb->sem);\r
                        break;\r
 \r
                default:\r
@@ -316,7 +320,7 @@ static int rping_cq_event_handler(struct rping_cb *cb)
 \r
 error:\r
        cb->state = RDMA_ERROR;\r
-       sem_post(&cb->sem);\r
+//     sem_post(&cb->sem);\r
        return ret;\r
 }\r
 \r
@@ -337,7 +341,8 @@ static int rping_accept(struct rping_cb *cb)
                return ret;\r
        }\r
 \r
-       sem_wait(&cb->sem);\r
+       cm_thread(cb);\r
+//     sem_wait(&cb->sem);\r
        if (cb->state == RDMA_ERROR) {\r
                fprintf(stderr, "wait for CONNECTED state %d\n", cb->state);\r
                return -1;\r
@@ -542,56 +547,59 @@ err1:
        return ret;\r
 }\r
 \r
-static void *cm_thread(void *arg)\r
+static void cm_thread(void *arg)\r
 {\r
        struct rping_cb *cb = arg;\r
        struct rdma_cm_event *event;\r
        int ret;\r
 \r
-       while (1) {\r
-               ret = rdma_get_cm_event(cb->cm_channel, &event);\r
-               if (ret) {\r
-                       fprintf(stderr, "rdma_get_cm_event err %d\n", ret);\r
-                       exit(ret);\r
-               }\r
-               ret = rping_cma_event_handler(event->id, event);\r
-               rdma_ack_cm_event(event);\r
-               if (ret)\r
-                       exit(ret);\r
+//     while (1) {\r
+       ret = rdma_get_cm_event(cb->cm_channel, &event);\r
+       if (ret) {\r
+               fprintf(stderr, "rdma_get_cm_event err %d\n", ret);\r
+               return;\r
+//                     exit(ret);\r
        }\r
+       ret = rping_cma_event_handler(event->id, event);\r
+       rdma_ack_cm_event(event);\r
+//             if (ret)\r
+//                     exit(ret);\r
+//     }\r
 }\r
 \r
-static void *cq_thread(void *arg)\r
+static void cq_thread(void *arg)\r
 {\r
        struct rping_cb *cb = arg;\r
        struct ibv_cq *ev_cq;\r
        void *ev_ctx;\r
        int ret;\r
        \r
-       DEBUG_LOG("cq_thread started.\n");\r
+//     DEBUG_LOG("cq_thread started.\n");\r
 \r
-       while (1) {     \r
-               pthread_testcancel();\r
+//     while (1) {     \r
+//             pthread_testcancel();\r
 \r
-               ret = ibv_get_cq_event(cb->channel, &ev_cq, &ev_ctx);\r
-               if (ret) {\r
-                       fprintf(stderr, "Failed to get cq event!\n");\r
-                       pthread_exit(NULL);\r
-               }\r
-               if (ev_cq != cb->cq) {\r
-                       fprintf(stderr, "Unknown CQ!\n");\r
-                       pthread_exit(NULL);\r
-               }\r
-               ret = ibv_req_notify_cq(cb->cq, 0);\r
-               if (ret) {\r
-                       fprintf(stderr, "Failed to set notify!\n");\r
-                       pthread_exit(NULL);\r
-               }\r
-               ret = rping_cq_event_handler(cb);\r
-               ibv_ack_cq_events(cb->cq, 1);\r
-               if (ret)\r
-                       pthread_exit(NULL);\r
+       ret = ibv_get_cq_event(cb->channel, &ev_cq, &ev_ctx);\r
+       if (ret) {\r
+               fprintf(stderr, "Failed to get cq event!\n");\r
+               return;\r
+//             pthread_exit(NULL);\r
+       }\r
+       if (ev_cq != cb->cq) {\r
+               fprintf(stderr, "Unknown CQ!\n");\r
+               return;\r
+//             pthread_exit(NULL);\r
+       }\r
+       ret = ibv_req_notify_cq(cb->cq, 0);\r
+       if (ret) {\r
+               fprintf(stderr, "Failed to set notify!\n");\r
+               return;\r
+//             pthread_exit(NULL);\r
        }\r
+       ret = rping_cq_event_handler(cb);\r
+       ibv_ack_cq_events(cb->cq, 1);\r
+//             pthread_exit(NULL);\r
+//     }\r
 }\r
 \r
 static void rping_format_send(struct rping_cb *cb, char *buf, struct ibv_mr *mr)\r
@@ -613,7 +621,8 @@ static int rping_test_server(struct rping_cb *cb)
 \r
        while (1) {\r
                /* Wait for client's Start STAG/TO/Len */\r
-               sem_wait(&cb->sem);\r
+               cq_thread(cb);\r
+//             sem_wait(&cb->sem);\r
                if (cb->state != RDMA_READ_ADV) {\r
                        fprintf(stderr, "wait for RDMA_READ_ADV state %d\n",\r
                                cb->state);\r
@@ -637,7 +646,8 @@ static int rping_test_server(struct rping_cb *cb)
                DEBUG_LOG("server posted rdma read req \n");\r
 \r
                /* Wait for read completion */\r
-               sem_wait(&cb->sem);\r
+               cq_thread(cb);\r
+//             sem_wait(&cb->sem);\r
                if (cb->state != RDMA_READ_COMPLETE) {\r
                        fprintf(stderr, "wait for RDMA_READ_COMPLETE state %d\n",\r
                                cb->state);\r
@@ -659,7 +669,8 @@ static int rping_test_server(struct rping_cb *cb)
                DEBUG_LOG("server posted go ahead\n");\r
 \r
                /* Wait for client's RDMA STAG/TO/Len */\r
-               sem_wait(&cb->sem);\r
+               cq_thread(cb);\r
+//             sem_wait(&cb->sem);\r
                if (cb->state != RDMA_WRITE_ADV) {\r
                        fprintf(stderr, "wait for RDMA_WRITE_ADV state %d\n",\r
                                cb->state);\r
@@ -685,7 +696,8 @@ static int rping_test_server(struct rping_cb *cb)
                }\r
 \r
                /* Wait for completion */\r
-               ret = sem_wait(&cb->sem);\r
+               cq_thread(cb);\r
+//             ret = sem_wait(&cb->sem);\r
                if (cb->state != RDMA_WRITE_COMPLETE) {\r
                        fprintf(stderr, "wait for RDMA_WRITE_COMPLETE state %d\n",\r
                                cb->state);\r
@@ -752,7 +764,8 @@ static int rping_run_server(struct rping_cb *cb)
        if (ret)\r
                return ret;\r
 \r
-       sem_wait(&cb->sem);\r
+       cm_thread(cb);\r
+//     sem_wait(&cb->sem);\r
        if (cb->state != CONNECT_REQUEST) {\r
                fprintf(stderr, "wait for CONNECT_REQUEST state %d\n",\r
                        cb->state);\r
@@ -777,7 +790,7 @@ static int rping_run_server(struct rping_cb *cb)
                goto err2;\r
        }\r
 \r
-       pthread_create(&cb->cqthread, NULL, cq_thread, cb);\r
+//     pthread_create(&cb->cqthread, NULL, cq_thread, cb);\r
 \r
        ret = rping_accept(cb);\r
        if (ret) {\r
@@ -827,7 +840,8 @@ static int rping_test_client(struct rping_cb *cb)
                }\r
 \r
                /* Wait for server to ACK */\r
-               sem_wait(&cb->sem);\r
+               cq_thread(cb);\r
+//             sem_wait(&cb->sem);\r
                if (cb->state != RDMA_WRITE_ADV) {\r
                        fprintf(stderr, "wait for RDMA_WRITE_ADV state %d\n",\r
                                cb->state);\r
@@ -843,7 +857,8 @@ static int rping_test_client(struct rping_cb *cb)
                }\r
 \r
                /* Wait for the server to say the RDMA Write is complete. */\r
-               sem_wait(&cb->sem);\r
+               cq_thread(cb);\r
+//             sem_wait(&cb->sem);\r
                if (cb->state != RDMA_WRITE_COMPLETE) {\r
                        fprintf(stderr, "wait for RDMA_WRITE_COMPLETE state %d\n",\r
                                cb->state);\r
@@ -881,7 +896,8 @@ static int rping_connect_client(struct rping_cb *cb)
                return ret;\r
        }\r
 \r
-       sem_wait(&cb->sem);\r
+       cm_thread(cb);\r
+//     sem_wait(&cb->sem);\r
        if (cb->state != CONNECTED) {\r
                fprintf(stderr, "wait for CONNECTED state %d\n", cb->state);\r
                return -1;\r
@@ -902,7 +918,8 @@ static int rping_bind_client(struct rping_cb *cb)
                return ret;\r
        }\r
 \r
-       sem_wait(&cb->sem);\r
+       cm_thread(cb);\r
+//     sem_wait(&cb->sem);\r
        if (cb->state != ROUTE_RESOLVED) {\r
                fprintf(stderr, "waiting for addr/route resolution state %d\n",\r
                        cb->state);\r
@@ -940,7 +957,7 @@ static int rping_run_client(struct rping_cb *cb)
                goto err2;\r
        }\r
 \r
-       pthread_create(&cb->cqthread, NULL, cq_thread, cb);\r
+//     pthread_create(&cb->cqthread, NULL, cq_thread, cb);\r
 \r
        ret = rping_connect_client(cb);\r
        if (ret) {\r
@@ -997,7 +1014,7 @@ static void usage()
 \r
 int main(int argc, char *argv[])\r
 {\r
-       struct rping_cb *cb;\r
+//     struct rping_cb *cb;\r
        int op;\r
        int ret = 0;\r
 \r
@@ -1011,7 +1028,7 @@ int main(int argc, char *argv[])
        cb->size = 64;\r
        cb->sin.sin_family = PF_INET;\r
        cb->port = htons(7174);\r
-       sem_init(&cb->sem, 0, 0);\r
+//     sem_init(&cb->sem, 0, 0);\r
 \r
        opterr = 0;\r
        while ((op=getopt(argc, argv, "a:Pp:C:S:t:scvVd")) != -1) {\r
@@ -1088,7 +1105,7 @@ int main(int argc, char *argv[])
        }\r
        DEBUG_LOG("created cm_id %p\n", cb->cm_id);\r
 \r
-       pthread_create(&cb->cmthread, NULL, cm_thread, cb);\r
+//     pthread_create(&cb->cmthread, NULL, cm_thread, cb);\r
 \r
        if (cb->server)\r
                ret = rping_run_server(cb);\r