From f2c214647b5ce53e52052d6b6bea3fbace7cc20a Mon Sep 17 00:00:00 2001 From: Arlin Davis Date: Sat, 20 Sep 2008 16:02:00 -0700 Subject: [PATCH] dtestx: Add new options to test UD. - many to one/many EP remote AH resolution, data flow - bi-directional EP remote AH resolution, data flow Signed-off by: Arlin Davis ardavis@ichips.intel.com --- test/dtest/dtestx.c | 650 ++++++++++++++++++++++++++++++-------------- 1 file changed, 441 insertions(+), 209 deletions(-) diff --git a/test/dtest/dtestx.c b/test/dtest/dtestx.c index fb89364..cfe00cd 100755 --- a/test/dtest/dtestx.c +++ b/test/dtest/dtestx.c @@ -75,6 +75,8 @@ int disconnect_ep(void); dat_strerror(status, &maj_msg, &min_msg);\ fprintf(stderr, str " returned %s : %s\n", maj_msg, min_msg);\ exit(1);\ + } else if (verbose) {\ + printf("dtestx: %s success\n",str);\ }\ } @@ -85,6 +87,8 @@ int disconnect_ep(void); dat_strerror(status, &maj_msg, &min_msg);\ fprintf(stderr, str " returned %s : %s\n", maj_msg, min_msg);\ exit(1);\ + } else if (verbose) {\ + printf("dtestx: %s\n",str);\ }\ } @@ -119,10 +123,14 @@ int disconnect_ep(void); #define ntoh64(x) (x) #endif /* __BYTE_ORDER == __BIG_ENDIAN */ +#define MIN(a, b) ((a < b) ? (a) : (b)) +#define MAX(a, b) ((a > b) ? (a) : (b)) + #define DTO_TIMEOUT (1000*1000*5) -#define CONN_TIMEOUT (1000*1000*10) -#define SERVER_TIMEOUT (1000*1000*120) -#define SERVER_CONN_QUAL 31111 +#define CONN_TIMEOUT (1000*1000*30) +#define SERVER_TIMEOUT (DAT_TIMEOUT_INFINITE) +#define CLIENT_ID 31111 +#define SERVER_ID 31112 #define BUF_SIZE 256 #define BUF_SIZE_ATOMIC 8 #define REG_MEM_COUNT 10 @@ -130,6 +138,7 @@ int disconnect_ep(void); #define RCV_RDMA_BUF_INDEX 1 #define SEND_BUF_INDEX 2 #define RECV_BUF_INDEX 3 +#define MAX_EP_COUNT 8 DAT_VADDR *atomic_buf; DAT_LMR_HANDLE lmr_atomic; @@ -137,14 +146,14 @@ DAT_LMR_CONTEXT lmr_atomic_context; DAT_RMR_CONTEXT rmr_atomic_context; DAT_VLEN reg_atomic_size; DAT_VADDR reg_atomic_addr; -DAT_LMR_HANDLE lmr[ REG_MEM_COUNT ]; -DAT_LMR_CONTEXT lmr_context[ REG_MEM_COUNT ]; -DAT_RMR_TRIPLET rmr[ REG_MEM_COUNT ]; -DAT_RMR_CONTEXT rmr_context[ REG_MEM_COUNT ]; -DAT_VLEN reg_size[ REG_MEM_COUNT ]; -DAT_VADDR reg_addr[ REG_MEM_COUNT ]; -DAT_RMR_TRIPLET * buf[ REG_MEM_COUNT ]; -DAT_EP_HANDLE ep; +DAT_LMR_HANDLE lmr[ REG_MEM_COUNT*MAX_EP_COUNT ]; +DAT_LMR_CONTEXT lmr_context[ REG_MEM_COUNT*MAX_EP_COUNT ]; +DAT_RMR_TRIPLET rmr[ REG_MEM_COUNT*MAX_EP_COUNT ]; +DAT_RMR_CONTEXT rmr_context[ REG_MEM_COUNT*MAX_EP_COUNT ]; +DAT_VLEN reg_size[ REG_MEM_COUNT*MAX_EP_COUNT ]; +DAT_VADDR reg_addr[ REG_MEM_COUNT*MAX_EP_COUNT ]; +DAT_RMR_TRIPLET * buf[ REG_MEM_COUNT*MAX_EP_COUNT ]; +DAT_EP_HANDLE ep[MAX_EP_COUNT]; DAT_EVD_HANDLE async_evd = DAT_HANDLE_NULL; DAT_IA_HANDLE ia = DAT_HANDLE_NULL; DAT_PZ_HANDLE pz = DAT_HANDLE_NULL; @@ -152,21 +161,30 @@ DAT_EVD_HANDLE cr_evd = DAT_HANDLE_NULL; DAT_EVD_HANDLE con_evd = DAT_HANDLE_NULL; DAT_EVD_HANDLE dto_evd = DAT_HANDLE_NULL; DAT_PSP_HANDLE psp = DAT_HANDLE_NULL; -DAT_CR_HANDLE cr = DAT_HANDLE_NULL; int server = 1; int ud_test = 0; +int multi_eps = 0; int buf_size = BUF_SIZE; int msg_size = sizeof(DAT_RMR_TRIPLET); char provider[64] = DAPL_PROVIDER; char hostname[256] = { 0 }; -DAT_IB_ADDR_HANDLE remote_ah; +DAT_IB_ADDR_HANDLE remote_ah[MAX_EP_COUNT]; +int eps = 1; +int verbose = 0; + +#define LOGPRINTF if (verbose) printf void print_usage(void) { printf("\n dtestx usage \n\n"); + printf("v: verbose\n"); printf("u unreliable datagram test\n"); + printf("U: unreliable datagram test, UD endpoint count\n"); + printf("m unreliable datagram test, multiple Server endpoints\n"); printf("b: buf length to allocate\n"); - printf("h: hostname/address of server, specified on client\n"); + printf("h: hostname/address of Server, specified on Client\n"); + printf("c: Client\n"); + printf("s: Server, default\n"); printf("P: provider name (default = ofa-v2-ib0)\n"); printf("\n"); } @@ -193,6 +211,7 @@ send_msg( DAT_EVENT event; DAT_COUNT nmore; DAT_RETURN status; + int i,ep_idx=0,ah_idx=0; DAT_DTO_COMPLETION_EVENT_DATA *dto_event = &event.event_data.dto_completion_event_data; @@ -200,26 +219,171 @@ send_msg( iov.virtual_address = (DAT_VADDR)data; iov.segment_length = (DAT_VLEN)size; - if (ud_test) - status = dat_ib_post_send_ud(ep, 1, &iov, - &remote_ah, cookie, flags); - else - status = dat_ep_post_send(ep, 1, &iov, cookie, flags); + for (i=0;isin_addr)); + + /* client expects all data in on first EP */ + status = dat_ib_post_send_ud(ep[ep_idx], + 1, + &iov, + &remote_ah[ah_idx], + cookie, + flags); + + } else { + status = dat_ep_post_send(ep[0], 1, &iov, + cookie, flags); + } + _OK(status, "dat_ep_post_send"); + + if (!(flags & DAT_COMPLETION_SUPPRESS_FLAG)) { + status = dat_evd_wait(dto_evd, DTO_TIMEOUT, + 1, &event, &nmore); + _OK(status, "dat_evd_wait after dat_ep_post_send"); + + if (event.event_number != DAT_DTO_COMPLETION_EVENT && + ud_test && event.event_number != + DAT_IB_DTO_EVENT) { + printf("unexpected event waiting post_send " + "completion - 0x%x\n", + event.event_number); + exit(1); + } + _OK(dto_event->status, "event status for post_send"); + } + } +} - _OK(status, "dat_ep_post_send"); +/* RC - Server only, UD - Server and Client, one per EP */ +void process_cr(int idx) +{ + DAT_EVENT event; + DAT_COUNT nmore; + DAT_RETURN status; + int pdata; + DAT_CR_HANDLE cr = DAT_HANDLE_NULL; + DAT_CONN_QUAL exp_qual = server?SERVER_ID:CLIENT_ID; + DAT_CR_PARAM cr_param; + DAT_CR_ARRIVAL_EVENT_DATA *cr_event = + &event.event_data.cr_arrival_event_data; + + LOGPRINTF("%s waiting for connect[%d] request\n", + server?"Server":"Client",idx); - if (!(flags & DAT_COMPLETION_SUPPRESS_FLAG)) { - status = dat_evd_wait(dto_evd, DTO_TIMEOUT, - 1, &event, &nmore); - _OK(status, "dat_evd_wait after dat_ep_post_send"); + status = dat_evd_wait(cr_evd, SERVER_TIMEOUT, 1, &event, &nmore); + _OK(status, "CR dat_evd_wait"); - if ((event.event_number != DAT_DTO_COMPLETION_EVENT) && - (ud_test && event.event_number != DAT_IB_DTO_EVENT)) { - printf("unexpected event waiting for post_send " - "completion - 0x%x\n", event.event_number); - exit(1); - } - _OK(dto_event->status, "event status for post_send"); + if (event.event_number != DAT_CONNECTION_REQUEST_EVENT && + (ud_test && event.event_number != + DAT_IB_UD_CONNECTION_REQUEST_EVENT)) { + printf("unexpected event,!conn req: 0x%x\n", + event.event_number); + exit(1); + } + + if ((cr_event->conn_qual != exp_qual) || + (cr_event->sp_handle.psp_handle != psp)) { + printf("wrong cr event data\n"); + exit(1); + } + + cr = cr_event->cr_handle; + status = dat_cr_query(cr, DAT_CSP_FIELD_ALL, &cr_param); + _OK(status, "dat_cr_query"); + + /* use private data to select EP */ + pdata = ntoh32(*((int*)cr_param.private_data)); + + LOGPRINTF("%s recvd pdata=0x%x, send pdata=0x%x\n", + server?"Server":"Client",pdata, + *(int*)cr_param.private_data); + + status = dat_cr_accept(cr, ep[pdata], 4, cr_param.private_data); + _OK(status, "dat_cr_accept"); + + printf("%s accepted CR on EP[%d]=%p\n", + server?"Server":"Client", + pdata, ep[pdata]); +} + +/* RC - Client and Server: 1, UD - Client: 1 per EP, Server: 2 per EP's */ +void process_conn(int idx) +{ + DAT_EVENT event; + DAT_COUNT nmore; + DAT_RETURN status; + int pdata; + DAT_IB_EXTENSION_EVENT_DATA *ext_event = + (DAT_IB_EXTENSION_EVENT_DATA *) + &event.event_extension_data[0]; + DAT_CONNECTION_EVENT_DATA *conn_event = + &event.event_data.connect_event_data; + + LOGPRINTF("%s waiting for connect[%d] establishment\n", + server?"Server":"Client",idx); + + status = dat_evd_wait(con_evd, CONN_TIMEOUT, 1, &event, &nmore); + _OK(status, "CONN dat_evd_wait"); + + LOGPRINTF("%s got connect[%d] event, pdata %p sz=%d\n", + server?"Server":"Client",idx, + conn_event->private_data, + conn_event->private_data_size); + + /* Waiting on CR's or CONN_EST */ + if (event.event_number != DAT_CONNECTION_EVENT_ESTABLISHED && + (ud_test && event.event_number != + DAT_IB_UD_CONNECTION_EVENT_ESTABLISHED)) { + printf("unexpected event, !conn established: 0x%x\n", + event.event_number); + exit(1); + } + + /* RC or PASSIVE CONN_EST we are done */ + if (!ud_test) + return; + + /* store each remote_ah according to remote EP index */ + pdata = ntoh32(*((int*)conn_event->private_data)); + LOGPRINTF(" Client got private data=0x%x\n", pdata); + + /* UD, get AH for sends. + * NOTE: bi-directional AH resolution results in a CONN_EST + * for both outbound connect and inbound CR. + * Use Active CONN_EST which includes server's CR + * pdata for remote_ah idx to send on and ignore PASSIVE CONN_EST. + * + * DAT_IB_UD_PASSIVE_REMOTE_AH == passive side CONN_EST + * DAT_IB_UD_REMOTE_AH == active side CONN_EST + */ + if (ext_event->type == DAT_IB_UD_REMOTE_AH) { + remote_ah[pdata] = ext_event->remote_ah; + printf("remote_ah[%d]: ah=%p, qpn=0x%x " + "addr=%s\n", + pdata, remote_ah[pdata].ah, + remote_ah[pdata].qpn, + inet_ntoa(((struct sockaddr_in*) + &remote_ah[pdata].ia_addr)->sin_addr)); + + } else if (ext_event->type != DAT_IB_UD_PASSIVE_REMOTE_AH) { + printf("unexpected UD ext_event type: 0x%x\n", + ext_event->type); + exit(1); } } @@ -235,28 +399,37 @@ connect_ep(char *hostname) DAT_LMR_TRIPLET iov; DAT_RMR_TRIPLET *r_iov; DAT_DTO_COOKIE cookie; - int i; - DAT_CR_ARRIVAL_EVENT_DATA *cr_event = - &event.event_data.cr_arrival_event_data; - DAT_DTO_COMPLETION_EVENT_DATA *dto_event = + DAT_CONN_QUAL conn_qual; + int i,ii,pdata,ctx; + DAT_PROVIDER_ATTR prov_attrs; + DAT_DTO_COMPLETION_EVENT_DATA *dto_event = &event.event_data.dto_completion_event_data; - DAT_IB_EXTENSION_EVENT_DATA *ext_event = - (DAT_IB_EXTENSION_EVENT_DATA *) - &event.event_extension_data[0]; - + status = dat_ia_open(provider, 8, &async_evd, &ia); _OK(status, "dat_ia_open"); - + + memset(&prov_attrs, 0, sizeof(prov_attrs)); + status = dat_ia_query(ia, NULL, 0, NULL, + DAT_PROVIDER_FIELD_ALL, &prov_attrs); + _OK(status, "dat_ia_query"); + + /* Print provider specific attributes */ + for (i=0;iconn_qual != SERVER_CONN_QUAL) || - (cr_event->sp_handle.psp_handle != psp)) { - printf("wrong cr event data\n"); - exit(1); - } - - cr = cr_event->cr_handle; - status = dat_cr_accept(cr, ep, 0, (DAT_PVOID)0); - printf("Server waiting for accept response\n"); - - } else { + } + + /* ud can resolve_ah and connect both ways */ + if (!server || (server && ud_test)) { struct addrinfo *target; if (getaddrinfo (hostname, NULL, NULL, &target) != 0) { @@ -383,52 +559,58 @@ connect_ep(char *hostname) exit(1); } - printf ("Server Name: %s \n", hostname); - printf ("Server Net Address: %s\n", inet_ntoa( - ((struct sockaddr_in *)target->ai_addr)->sin_addr)); + printf ("Remote %s Name: %s \n", + server?"Client":"Server", hostname); + printf ("Remote %s Net Address: %s\n", + server?"Client":"Server", + inet_ntoa(((struct sockaddr_in *) + target->ai_addr)->sin_addr)); remote_addr = *((DAT_IA_ADDRESS_PTR)target->ai_addr); freeaddrinfo(target); - strcpy((char*)buf[ SND_RDMA_BUF_INDEX ],"client written data"); - status = dat_ep_connect(ep, - &remote_addr, - SERVER_CONN_QUAL, - CONN_TIMEOUT, - 0, - (DAT_PVOID)0, - 0, - DAT_CONNECT_DEFAULT_FLAG ); - _OK(status, "dat_psp_create"); - printf("Client waiting for connect response\n"); - } - - status = dat_evd_wait(con_evd, CONN_TIMEOUT, 1, &event, &nmore); - _OK(status, "connect dat_evd_wait"); - - if (event.event_number != DAT_CONNECTION_EVENT_ESTABLISHED && - (ud_test && event.event_number != - DAT_IB_UD_CONNECTION_EVENT_ESTABLISHED)) { - printf("unexpected event, !conn established: 0x%x\n", - event.event_number); - exit(1); + strcpy((char*)buf[SND_RDMA_BUF_INDEX],"Client written data"); + + /* one Client EP, multiple Server EPs, same conn_qual + * use private data to select EP on Server + */ + for (i=0;itype == DAT_IB_UD_REMOTE_AH) { - remote_ah = ext_event->remote_ah; - printf(" remote_ah: ah=%p, qpn=0x%x addr=%s\n", - remote_ah.ah, remote_ah.qpn, - inet_ntoa(((struct sockaddr_in *) - &remote_ah.ia_addr)->sin_addr)); - } else { - printf("unexpected UD ext_event type: 0x%x\n", - ext_event->type); - exit(1); - } + for (i=(server?1:0);ivirtual_address = hton64((DAT_VADDR)buf[RCV_RDMA_BUF_INDEX]); r_iov->segment_length = hton32(buf_size); - printf("%d Send RMR msg to remote: r_key_ctx=0x%x,va="F64x",len=0x%x\n", - getpid(), hton32(r_iov->rmr_context), - hton64(r_iov->virtual_address), hton32(r_iov->segment_length)); + printf("Send RMR message: r_key_ctx=0x%x,va="F64x",len=0x%x\n", + hton32(r_iov->rmr_context), + hton64(r_iov->virtual_address), + hton32(r_iov->segment_length)); send_msg(buf[SEND_BUF_INDEX], sizeof(DAT_RMR_TRIPLET), @@ -452,44 +635,69 @@ connect_ep(char *hostname) /* * Wait for their RMR */ - printf("Waiting for remote to send RMR data\n"); - status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore); - _OK(status, "dat_evd_wait after dat_ep_post_send"); + for (i=0,ctx=0;istatus, "event status for post_recv"); - - if (dto_event->transfered_length != msg_size || - dto_event->user_cookie.as_64 != RECV_BUF_INDEX) { - printf("unexpected event data for receive: len=%d cookie=%d " - "expected %d/%d\n", - (int)dto_event->transfered_length, - (int)dto_event->user_cookie.as_64, - msg_size, RECV_BUF_INDEX); - exit(1); - } - - /* swap RMR,address info to host order */ - if (ud_test) - r_iov = (DAT_RMR_TRIPLET*)((char*)buf[RECV_BUF_INDEX]+40); - else - r_iov = (DAT_RMR_TRIPLET*)buf[RECV_BUF_INDEX]; - - r_iov->rmr_context = ntoh32(r_iov->rmr_context); - r_iov->virtual_address = ntoh64(r_iov->virtual_address); - r_iov->segment_length = ntoh32(r_iov->segment_length); + status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore); + _OK(status, "dat_evd_wait after dat_ep_post_send"); - printf("%d Received RMR from remote: " - "r_iov: r_key_ctx=%x,va="F64x",len=0x%x\n", - getpid(), r_iov->rmr_context, - r_iov->virtual_address, - r_iov->segment_length); + if ((event.event_number != DAT_DTO_COMPLETION_EVENT) && + (ud_test && event.event_number != DAT_IB_DTO_EVENT)) { + printf("unexpected event waiting for RMR context " + "- 0x%x\n", event.event_number); + exit(1); + } + _OK(dto_event->status, "event status for post_recv"); + + /* careful when checking cookies: + * Client - receiving multi messages on a single EP + * Server - not receiving on multiple EP's + */ + if (!server || (server && !multi_eps)) { + if (dto_event->transfered_length != msg_size || + dto_event->user_cookie.as_64 != ctx) { + printf("unexpected event data on recv: len=%d" + " cookie="F64x" expected %d/%d\n", + (int)dto_event->transfered_length, + dto_event->user_cookie.as_64, + msg_size, ctx); + exit(1); + } + /* Server - receiving one message each across many EP's */ + } else { + if (dto_event->transfered_length != msg_size || + dto_event->user_cookie.as_64 != RECV_BUF_INDEX) { + printf("unexpected event data on recv: len=%d" + "cookie="F64x" expected %d/%d\n", + (int)dto_event->transfered_length, + dto_event->user_cookie.as_64, + msg_size, RECV_BUF_INDEX); + exit(1); + } + } + /* swap RMR,address info to host order */ + if (!server || (server && !multi_eps)) + r_iov = (DAT_RMR_TRIPLET*)buf[ctx]; + else + r_iov = (DAT_RMR_TRIPLET*)buf[(i*REG_MEM_COUNT)+RECV_BUF_INDEX]; + + if (ud_test) + r_iov = (DAT_RMR_TRIPLET*)((char*)r_iov + 40); + + r_iov->rmr_context = ntoh32(r_iov->rmr_context); + r_iov->virtual_address = ntoh64(r_iov->virtual_address); + r_iov->segment_length = ntoh32(r_iov->segment_length); + + printf("Recv RMR message: r_iov(%p):" + " r_key_ctx=%x,va="F64x",len=0x%x on EP=%p\n", + r_iov, r_iov->rmr_context, + r_iov->virtual_address, + r_iov->segment_length, + dto_event->ep_handle); + } return(0); } @@ -497,42 +705,43 @@ int disconnect_ep(void) { DAT_RETURN status; - int i; DAT_EVENT event; DAT_COUNT nmore; + int i; - status = dat_ep_disconnect(ep, DAT_CLOSE_DEFAULT); - _OK2(status, "dat_ep_disconnect"); - - status = dat_evd_wait(con_evd, DAT_TIMEOUT_INFINITE, 1, - &event, &nmore); - _OK(status, "dat_evd_wait"); - - if (server) { + if (!ud_test) { + status = dat_ep_disconnect(ep[0], DAT_CLOSE_DEFAULT); + _OK2(status, "dat_ep_disconnect"); + + status = dat_evd_wait(con_evd, DAT_TIMEOUT_INFINITE, 1, + &event, &nmore); + _OK(status, "dat_evd_wait"); + } + if (psp) { status = dat_psp_free(psp); _OK2(status, "dat_psp_free"); } - - for (i = 0; i < REG_MEM_COUNT; i++) { + for (i = 0; i < REG_MEM_COUNT*eps; i++) { status = dat_lmr_free(lmr[ i ]); _OK2(status, "dat_lmr_free"); } - if (lmr_atomic) { status = dat_lmr_free(lmr_atomic); _OK2(status, "dat_lmr_free_atomic"); } - - status = dat_ep_free(ep); - _OK2(status, "dat_ep_free"); - + for (i=0;i