/*\r
- * Copyright (c) 2007 Intel Corporation. All rights reserved.\r
+ * Copyright (c) 2005-2008 Intel Corporation. All rights reserved.\r
*\r
* This software is available to you under a choice of one of two\r
* licenses. You may choose to be licensed under the terms of the GNU\r
#include <stdlib.h>\r
#include <string.h>\r
\r
+#ifdef DAPL_PROVIDER\r
+#undef DAPL_PROVIDER\r
+#endif\r
+\r
#if defined(_WIN32) || defined(_WIN64)\r
\r
#include <windows.h>\r
#include <io.h>\r
#include <process.h>\r
#include <getopt.h>\r
-#include <complib\cl_byteswap.h>\r
+#include <complib/cl_types.h>\r
\r
#define getpid _getpid\r
-#define PRIx64 "%I64x" \r
+#define F64x "%I64x" \r
+\r
+#ifdef DBG\r
+#define DAPL_PROVIDER "ibnic0v2d"\r
+#else\r
+#define DAPL_PROVIDER "ibnic0v2"\r
+#endif\r
\r
#else\r
\r
+#include <endian.h>\r
+#include <byteswap.h>\r
#include <netdb.h>\r
#include <sys/socket.h>\r
#include <netinet/in.h>\r
#include <getopt.h>\r
#include <inttypes.h>\r
\r
-#endif\r
+#define DAPL_PROVIDER "ofa-v2-ib0"\r
+\r
+#define F64x "%"PRIx64""\r
+#endif \r
+\r
+/* byte swap helpers from Complib */\r
+#if __BYTE_ORDER == __LITTLE_ENDIAN\r
+#define ntoh16(x) (uint16_t)( \\r
+ (((uint16_t)(x) & 0x00FF) << 8) | \\r
+ (((uint16_t)(x) & 0xFF00) >> 8))\r
+#define hton16(x) ntoh16(x)\r
+#define ntoh32(x) (uint32_t)( \\r
+ (((uint32_t)(x) & 0x000000FF) << 24)| \\r
+ (((uint32_t)(x) & 0x0000FF00) << 8) | \\r
+ (((uint32_t)(x) & 0x00FF0000) >> 8) | \\r
+ (((uint32_t)(x) & 0xFF000000) >> 24))\r
+#define hton32(x) ntoh32(x)\r
+#define ntoh64(x) (uint64_t)( \\r
+ (((uint64_t)x & 0x00000000000000FF) << 56) | \\r
+ (((uint64_t)x & 0x000000000000FF00) << 40) | \\r
+ (((uint64_t)x & 0x0000000000FF0000) << 24) | \\r
+ (((uint64_t)x & 0x00000000FF000000) << 8 ) | \\r
+ (((uint64_t)x & 0x000000FF00000000) >> 8 ) | \\r
+ (((uint64_t)x & 0x0000FF0000000000) >> 24) | \\r
+ (((uint64_t)x & 0x00FF000000000000) >> 40) | \\r
+ (((uint64_t)x & 0xFF00000000000000) >> 56))\r
+#define hton64(x) ntoh64(x)\r
+#elif __BYTE_ORDER == __BIG_ENDIAN\r
+#define hton16(x) (x)\r
+#define ntoh16(x) (x)\r
+#define hton32(x) (x)\r
+#define ntoh32(x) (x)\r
+#define hton64(x) (x)\r
+#define ntoh64(x) (x)\r
+#endif /* __BYTE_ORDER == __BIG_ENDIAN */\r
\r
/* Debug: 1 == connect & close only, otherwise full-meal deal */\r
#define CONNECT_ONLY 0\r
\r
-#ifdef DAPL_PROVIDER\r
-#undef DAPL_PROVIDER\r
-#endif\r
-#ifdef DBG\r
-#define DAPL_PROVIDER "ibnic0v2d"\r
-#else\r
-#define DAPL_PROVIDER "ibnic0v2"\r
-#endif\r
-\r
#define MAX_POLLING_CNT 50000\r
#define MAX_RDMA_RD 4\r
#define MAX_PROCS 1000\r
#define DTO_TIMEOUT (1000*1000*5)\r
#define DTO_FLUSH_TIMEOUT (1000*1000*2)\r
#define CONN_TIMEOUT (1000*1000*10)\r
-#define SERVER_TIMEOUT (1000*1000*20)\r
+#define SERVER_TIMEOUT DAT_TIMEOUT_INFINITE\r
#define RDMA_BUFFER_SIZE (64)\r
\r
/* Global DAT vars */\r
static DAT_VADDR registered_addr_send_msg;\r
static DAT_EP_ATTR ep_attr;\r
char hostname[256] = {0};\r
-char provider[64] = DAPL_PROVIDER;\r
+char provider[64] = DAPL_PROVIDER;\r
+char addr_str[INET_ADDRSTRLEN];\r
\r
/* rdma pointers */\r
char *rbuf = NULL;\r
\r
/* defaults */\r
static int parent=1;\r
+static int failed=0;\r
+static int performance_times=0;\r
static int connected=0;\r
static int burst=10;\r
static int server=1;\r
DAT_DTO_COOKIE cookie,\r
DAT_COMPLETION_FLAGS flags );\r
\r
-DAT_RETURN connect_ep( char *hostname, int conn_id );\r
+DAT_RETURN connect_ep( char *hostname, DAT_CONN_QUAL conn_id );\r
void disconnect_ep( void );\r
DAT_RETURN register_rdma_memory( void );\r
DAT_RETURN unregister_rdma_memory( void );\r
DAT_RETURN do_rdma_read_with_msg( void );\r
DAT_RETURN do_ping_pong_msg( void );\r
\r
-#define LOGPRINTF1(_format, _aa) if (verbose) printf(_format, _aa)\r
-#define LOGPRINTF2(_format, _aa,_bb) if (verbose) printf(_format, _aa,_bb)\r
-#define LOGPRINTF3(_format, _aa,_bb,_cc) \\r
- if (verbose) printf(_format, _aa,_bb,_cc)\r
-#define LOGPRINTF4(_format, _aa,_bb,_cc,_dd) \\r
- if (verbose) printf(_format, _aa,_bb,_cc,_dd)\r
-#define LOGPRINTF5(_format, _aa,_bb,_cc,_dd,_ee) \\r
- if (verbose) printf(_format, _aa,_bb,_cc,_dd,_ee)\r
-\r
+#define LOGPRINTF if (verbose) printf\r
\r
main(int argc, char **argv)\r
{\r
int i,c;\r
DAT_RETURN ret;\r
+ DAT_EP_PARAM ep_param;\r
\r
/* parse arguments */\r
- while ((c = getopt(argc, argv, "scvpb:d:B:h:P:")) != -1)\r
+ while ((c = getopt(argc, argv, "tscvpb:d:B:h:P:")) != -1)\r
{\r
switch(c)\r
{\r
+ case 't':\r
+ performance_times = 1;\r
+ fflush(stdout);\r
+ break;\r
case 's':\r
server = 1;\r
fflush(stdout);\r
exit(1);\r
}\r
memset( &time, 0, sizeof(struct dt_time) );\r
- LOGPRINTF4("%d Allocated RDMA buffers (r:%p,s:%p) len %d \n",\r
+ LOGPRINTF("%d Allocated RDMA buffers (r:%p,s:%p) len %d \n",\r
getpid(), rbuf, sbuf, buf_len);\r
\r
/* dat_ia_open, dat_pz_create */\r
getpid(),DT_RetToString(ret));\r
exit(1);\r
} else\r
- LOGPRINTF1("%d Opened Interface Adaptor\n",getpid());\r
+ LOGPRINTF("%d Opened Interface Adaptor\n",getpid());\r
\r
/* Create Protection Zone */\r
start = get_time();\r
- LOGPRINTF1("%d Create Protection Zone\n",getpid());\r
+ LOGPRINTF("%d Create Protection Zone\n",getpid());\r
ret = dat_pz_create(h_ia, &h_pz);\r
stop = get_time();\r
time.pzc += ((stop - start)*1.0e6);\r
getpid(),DT_RetToString(ret));\r
exit(1);\r
} else\r
- LOGPRINTF1("%d Created Protection Zone\n",getpid());\r
+ LOGPRINTF("%d Created Protection Zone\n",getpid());\r
\r
/* Register memory */\r
- LOGPRINTF1("%d Register RDMA memory\n", getpid());\r
+ LOGPRINTF("%d Register RDMA memory\n", getpid());\r
ret = register_rdma_memory();\r
if(ret != DAT_SUCCESS) {\r
fprintf(stderr, "%d Error registering RDMA memory: %s\n",\r
getpid(),DT_RetToString(ret));\r
goto cleanup;\r
} else\r
- LOGPRINTF1("%d Register RDMA memory done\n", getpid());\r
+ LOGPRINTF("%d Register RDMA memory done\n", getpid());\r
\r
- LOGPRINTF1("%d Create events\n", getpid());\r
+ LOGPRINTF("%d Create events\n", getpid());\r
ret = create_events();\r
if(ret != DAT_SUCCESS) {\r
fprintf(stderr, "%d Error creating events: %s\n",\r
getpid(),DT_RetToString(ret));\r
goto cleanup;\r
} else {\r
- LOGPRINTF1("%d Create events done\n", getpid());\r
+ LOGPRINTF("%d Create events done\n", getpid());\r
}\r
\r
/* create EP */\r
getpid(),DT_RetToString(ret));\r
goto cleanup;\r
} else\r
- LOGPRINTF2("%d EP created %p \n", getpid(), h_ep);\r
+ LOGPRINTF("%d EP created %p \n", getpid(), h_ep);\r
\r
/*\r
* register message buffers, establish connection, and\r
getpid(),DT_RetToString(ret));\r
goto cleanup;\r
} else\r
- LOGPRINTF1("%d connect_ep complete\n", getpid());\r
+ LOGPRINTF("%d connect_ep complete\n", getpid());\r
+\r
+ /* Query EP for local and remote address information, print */\r
+ ret = dat_ep_query(h_ep, DAT_EP_FIELD_ALL, &ep_param);\r
+ if(ret != DAT_SUCCESS) {\r
+ fprintf(stderr, "%d Error dat_ep_query: %s\n",\r
+ getpid(),DT_RetToString(ret));\r
+ goto cleanup;\r
+ } else\r
+ LOGPRINTF("%d EP queried %p \n", getpid(), h_ep);\r
+#if defined(_WIN32)\r
+ printf("\n%d Query EP: LOCAL addr %s port %lld\n", getpid(),\r
+ inet_ntoa(((struct sockaddr_in *)\r
+ ep_param.local_ia_address_ptr)->sin_addr),\r
+ (ep_param.local_port_qual));\r
+#else\r
+ inet_ntop(AF_INET,\r
+ &((struct sockaddr_in *)ep_param.local_ia_address_ptr)->sin_addr,\r
+ addr_str, sizeof(addr_str));\r
+ printf("\n%d Query EP: LOCAL addr %s port %lld\n", getpid(),\r
+ addr_str, (ep_param.local_port_qual));\r
+#endif\r
+#if defined(_WIN32)\r
+ printf("%d Query EP: REMOTE addr %s port %lld\n", getpid(),\r
+ inet_ntoa(((struct sockaddr_in *)\r
+ ep_param.local_ia_address_ptr)->sin_addr),\r
+ (ep_param.remote_port_qual));\r
+#else\r
+ inet_ntop(AF_INET,\r
+ &((struct sockaddr_in *)ep_param.remote_ia_address_ptr)->sin_addr,\r
+ addr_str, sizeof(addr_str));\r
+ printf("%d Query EP: REMOTE addr %s port %lld\n", getpid(),\r
+ addr_str, (ep_param.remote_port_qual));\r
+#endif\r
+ fflush(stdout);\r
\r
#if CONNECT_ONLY\r
#if defined(_WIN32) || defined(_WIN64)\r
getpid(),DT_RetToString(ret));\r
goto cleanup;\r
} else\r
- LOGPRINTF1("%d do_rdma_write_with_msg complete\n", getpid());\r
+ LOGPRINTF("%d do_rdma_write_with_msg complete\n", getpid());\r
\r
/*********** RDMA read data *************/\r
ret = do_rdma_read_with_msg();\r
getpid(),DT_RetToString(ret));\r
goto cleanup;\r
} else\r
- LOGPRINTF1("%d do_rdma_read_with_msg complete\n", getpid());\r
+ LOGPRINTF("%d do_rdma_read_with_msg complete\n", getpid());\r
\r
/*********** PING PING messages ************/\r
ret = do_ping_pong_msg();\r
fprintf(stderr, "%d Error do_ping_pong_msg: %s\n",\r
getpid(),DT_RetToString(ret));\r
goto cleanup;\r
- } else\r
- LOGPRINTF1("%d do_ping_pong_msg complete\n", getpid());\r
+ } else {\r
+ LOGPRINTF("%d do_ping_pong_msg complete\n", getpid());\r
+ goto complete;\r
+ }\r
\r
cleanup:\r
+ failed++;\r
+complete:\r
+\r
/* disconnect and free EP resources */\r
if ( h_ep != DAT_HANDLE_NULL ) {\r
/* unregister message buffers and tear down connection */\r
- LOGPRINTF2("%d Disconnect and Free EP %p \n",getpid(),h_ep);\r
+ LOGPRINTF("%d Disconnect and Free EP %p \n",getpid(),h_ep);\r
disconnect_ep();\r
+ \r
+ /* free EP */\r
+ LOGPRINTF("%d Free EP %p \n",getpid(),h_ep);\r
+ start = get_time();\r
+ ret = dat_ep_free( h_ep );\r
+ stop = get_time();\r
+ time.epf += ((stop - start)*1.0e6);\r
+ time.total += time.epf;\r
+ if(ret != DAT_SUCCESS) {\r
+ fprintf(stderr, "%d Error freeing EP: %s\n",\r
+ getpid(), DT_RetToString(ret));\r
+ } else {\r
+ LOGPRINTF("%d Freed EP\n",getpid());\r
+ h_ep = DAT_HANDLE_NULL;\r
+ } \r
}\r
-\r
- /* free EP */\r
- LOGPRINTF2("%d Free EP %p \n",getpid(),h_ep);\r
- start = get_time();\r
- ret = dat_ep_free( h_ep );\r
- stop = get_time();\r
- time.epf += ((stop - start)*1.0e6);\r
- time.total += time.epf;\r
- if(ret != DAT_SUCCESS) {\r
- fprintf(stderr, "%d Error freeing EP: %s\n",\r
- getpid(), DT_RetToString(ret));\r
- } else {\r
- LOGPRINTF1("%d Freed EP\n",getpid());\r
- h_ep = DAT_HANDLE_NULL;\r
- }\r
-\r
+ \r
/* free EVDs */\r
- LOGPRINTF1("%d destroy events\n", getpid());\r
+ LOGPRINTF("%d destroy events\n", getpid());\r
ret = destroy_events();\r
if(ret != DAT_SUCCESS)\r
fprintf(stderr, "%d Error destroy_events: %s\n",\r
getpid(),DT_RetToString(ret));\r
else\r
- LOGPRINTF1("%d destroy events done\n", getpid());\r
+ LOGPRINTF("%d destroy events done\n", getpid());\r
\r
\r
ret = unregister_rdma_memory();\r
- LOGPRINTF1("%d unregister_rdma_memory \n", getpid());\r
+ LOGPRINTF("%d unregister_rdma_memory \n", getpid());\r
if(ret != DAT_SUCCESS)\r
fprintf(stderr, "%d Error unregister_rdma_memory: %s\n",\r
getpid(),DT_RetToString(ret));\r
else\r
- LOGPRINTF1("%d unregister_rdma_memory done\n", getpid());\r
+ LOGPRINTF("%d unregister_rdma_memory done\n", getpid());\r
\r
/* Free protection domain */\r
- LOGPRINTF1("%d Freeing pz\n",getpid());\r
+ LOGPRINTF("%d Freeing pz\n",getpid());\r
start = get_time();\r
ret = dat_pz_free( h_pz );\r
stop = get_time();\r
fprintf(stderr, "%d Error freeing PZ: %s\n",\r
getpid(), DT_RetToString(ret));\r
} else {\r
- LOGPRINTF1("%d Freed pz\n",getpid());\r
+ LOGPRINTF("%d Freed pz\n",getpid());\r
h_pz = NULL;\r
}\r
\r
/* close the device */\r
- LOGPRINTF1("%d Closing Interface Adaptor\n",getpid());\r
+ LOGPRINTF("%d Closing Interface Adaptor\n",getpid());\r
start = get_time();\r
//ret = dat_ia_close( h_ia, DAT_CLOSE_ABRUPT_FLAG );\r
ret = dat_ia_close( h_ia, DAT_CLOSE_GRACEFUL_FLAG );\r
getpid(),DT_RetToString(ret));\r
exit(1);\r
} else\r
- LOGPRINTF1("%d Closed Interface Adaptor\n",getpid());\r
+ LOGPRINTF("%d Closed Interface Adaptor\n",getpid());\r
+\r
+ /* free rdma buffers */\r
+ free(rbuf);\r
+ free(sbuf);\r
+\r
+ printf("\n%d: DAPL Test Complete. %s\n\n",\r
+ getpid(), failed?"FAILED":"PASSED");\r
+ \r
+ if (!performance_times) \r
+ exit(0);\r
\r
printf( "\n%d: DAPL Test Complete.\n\n",getpid());\r
printf( "%d: Message RTT: Total=%10.2lf usec, %d bursts, itime=%10.2lf"\r
printf( "%d: EP free: %10.2lf usec\n",getpid(), time.epf );\r
printf( "%d: TOTAL: %10.2lf usec\n",getpid(), time.total );\r
\r
- /* free rdma buffers */\r
- free(rbuf);\r
- free(sbuf);\r
#if defined(_WIN32) || defined(_WIN64)\r
WSACleanup();\r
#endif\r
DAT_RETURN ret;\r
\r
iov.lmr_context = context;\r
+#if defined(_WIN32)\r
iov.virtual_address = (DAT_VADDR)data;\r
+#else\r
+ iov.virtual_address = (DAT_VADDR)(unsigned long)data;\r
+#endif\r
iov.segment_length = size;\r
\r
- LOGPRINTF1("%d calling post_send\n", getpid());\r
+ LOGPRINTF("%d calling post_send\n", getpid());\r
cookie.as_64 = 0xaaaa;\r
ret = dat_ep_post_send( h_ep,\r
1,\r
while (dat_evd_dequeue(h_dto_req_evd, &event) == DAT_QUEUE_EMPTY) ;\r
}\r
else {\r
- LOGPRINTF1("%d waiting for post_send completion event\n", getpid());\r
+ LOGPRINTF("%d waiting for post_send completion event\n", getpid());\r
if (use_cno) {\r
DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;\r
ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd );\r
- LOGPRINTF2("%d cno wait return evd_handle=%p\n", getpid(),evd);\r
+ LOGPRINTF("%d cno wait return evd_handle=%p\n", getpid(),evd);\r
if ( evd != h_dto_req_evd ) {\r
fprintf(stderr,\r
"%d Error waiting on h_dto_cno: evd != h_dto_req_evd\n",\r
\r
if ((event.event_data.dto_completion_event_data.transfered_length != size ) ||\r
(event.event_data.dto_completion_event_data.user_cookie.as_64 != 0xaaaa )) {\r
- fprintf(stderr, "%d: ERROR: DTO len %d or cookie %llx\n",\r
+ fprintf(stderr, "%d: ERROR: DTO len %d or cookie "F64x" \n",\r
getpid(),\r
event.event_data.dto_completion_event_data.transfered_length,\r
event.event_data.dto_completion_event_data.user_cookie.as_64 );\r
\r
\r
DAT_RETURN\r
-connect_ep( char *hostname, int conn_id )\r
+connect_ep( char *hostname, DAT_CONN_QUAL conn_id )\r
{\r
- DAT_SOCK_ADDR remote_addr;\r
- DAT_RETURN ret;\r
- DAT_REGION_DESCRIPTION region;\r
- DAT_EVENT event;\r
- DAT_COUNT nmore;\r
- DAT_LMR_TRIPLET l_iov;\r
- DAT_RMR_TRIPLET r_iov;\r
- DAT_DTO_COOKIE cookie;\r
- int i;\r
- DAT_VADDR va; \r
- DAT_SEG_LENGTH len;\r
- DAT_RMR_CONTEXT rmr_ctx;\r
-\r
+ DAT_SOCK_ADDR remote_addr;\r
+ DAT_RETURN ret;\r
+ DAT_REGION_DESCRIPTION region;\r
+ DAT_EVENT event;\r
+ DAT_COUNT nmore;\r
+ DAT_LMR_TRIPLET l_iov;\r
+ DAT_RMR_TRIPLET r_iov;\r
+ DAT_DTO_COOKIE cookie;\r
+ int i;\r
+ \r
/* Register send message buffer */\r
- LOGPRINTF3("%d Registering send Message Buffer %p, len %d\n",\r
+ LOGPRINTF("%d Registering send Message Buffer %p, len %d\n",\r
getpid(), &rmr_send_msg, sizeof(DAT_RMR_TRIPLET) );\r
region.for_va = &rmr_send_msg;\r
ret = dat_lmr_create( h_ia,\r
return(ret);\r
}\r
else\r
- LOGPRINTF2("%d Registered send Message Buffer %p \n",\r
+ LOGPRINTF("%d Registered send Message Buffer %p \n",\r
getpid(),region.for_va );\r
\r
/* Register Receive buffers */\r
- LOGPRINTF2("%d Registering Receive Message Buffer %p\n",\r
+ LOGPRINTF("%d Registering Receive Message Buffer %p\n",\r
getpid(), rmr_recv_msg );\r
region.for_va = rmr_recv_msg;\r
ret = dat_lmr_create( h_ia,\r
return(ret);\r
}\r
else\r
- LOGPRINTF2("%d Registered Receive Message Buffer %p\n",\r
+ LOGPRINTF("%d Registered Receive Message Buffer %p\n",\r
getpid(),region.for_va);\r
\r
for ( i = 0; i < MSG_BUF_COUNT; i++ ) {\r
cookie.as_64 = i;\r
l_iov.lmr_context = lmr_context_recv_msg;\r
+#if defined(_WIN32)\r
l_iov.virtual_address = (DAT_VADDR)&rmr_recv_msg[ i ];\r
+#else\r
+ l_iov.virtual_address = (DAT_VADDR)(unsigned long)&rmr_recv_msg[ i ];\r
+#endif\r
l_iov.segment_length = sizeof(DAT_RMR_TRIPLET);\r
\r
- LOGPRINTF2("%d Posting Receive Message Buffer %p\n",\r
+ LOGPRINTF("%d Posting Receive Message Buffer %p\n",\r
getpid(), &rmr_recv_msg[ i ]);\r
ret = dat_ep_post_recv( h_ep,\r
1,\r
return(ret);\r
}\r
else\r
- LOGPRINTF2("%d Registered Receive Message Buffer %p\n",\r
+ LOGPRINTF("%d Registered Receive Message Buffer %p\n",\r
getpid(),region.for_va);\r
\r
}\r
if ( server ) { /* SERVER */\r
\r
/* create the service point for server listen */\r
- LOGPRINTF1("%d Creating service point for listen\n",getpid());\r
+ LOGPRINTF("%d Creating service point for listen\n",getpid());\r
ret = dat_psp_create( h_ia,\r
conn_id,\r
h_cr_evd,\r
return(ret);\r
}\r
else\r
- LOGPRINTF1("%d dat_psp_created for server listen\n", getpid());\r
+ LOGPRINTF("%d dat_psp_created for server listen\n", getpid());\r
\r
- printf("%d Server waiting for connect request..\n", getpid());\r
+ printf("%d Server waiting for connect request on port %lld\n", \r
+ getpid(),conn_id);\r
+ \r
ret = dat_evd_wait( h_cr_evd, SERVER_TIMEOUT, 1, &event, &nmore );\r
if(ret != DAT_SUCCESS) {\r
fprintf(stderr, "%d Error dat_evd_wait: %s\n",\r
return(ret);\r
}\r
else\r
- LOGPRINTF1("%d dat_evd_wait for cr_evd completed\n", getpid());\r
+ LOGPRINTF("%d dat_evd_wait for cr_evd completed\n", getpid());\r
\r
if ( event.event_number != DAT_CONNECTION_REQUEST_EVENT ) {\r
fprintf(stderr, "%d Error unexpected cr event : %s\n",\r
getpid(),DT_EventToSTr(event.event_number));\r
- return( DAT_ABORT );\r
+ return( DAT_ABORT );\r
}\r
if ( (event.event_data.cr_arrival_event_data.conn_qual != SERVER_CONN_QUAL) ||\r
(event.event_data.cr_arrival_event_data.sp_handle.psp_handle != h_psp) ) {\r
#endif\r
/* accept connect request from client */\r
h_cr = event.event_data.cr_arrival_event_data.cr_handle;\r
- LOGPRINTF1("%d Accepting connect request from client\n",getpid());\r
+ LOGPRINTF("%d Accepting connect request from client\n",getpid());\r
ret = dat_cr_accept( h_cr, h_ep, 0, (DAT_PVOID)0 );\r
- if(ret != DAT_SUCCESS) {\r
+ if(ret != DAT_SUCCESS) {\r
fprintf(stderr, "%d Error dat_cr_accept: %s\n",\r
getpid(),DT_RetToString(ret));\r
return(ret);\r
}\r
else\r
- LOGPRINTF1("%d dat_cr_accept completed\n", getpid());\r
+ LOGPRINTF("%d dat_cr_accept completed\n", getpid());\r
}\r
else { /* CLIENT */\r
struct addrinfo *target;\r
perror("\n remote name resolution failed!");\r
exit ( 1 );\r
}\r
- rval = ((struct sockaddr_in *)target->ai_addr)->sin_addr.sin_addr;\r
+ rval = ((struct sockaddr_in *)target->ai_addr)->sin_addr.s_addr;\r
#endif\r
printf ("%d Server Name: %s \n", getpid(), hostname);\r
- printf ("%d Server Net Address: %d.%d.%d.%d\n", getpid(),\r
- (rval >> 0) & 0xff,\r
- (rval >> 8) & 0xff,\r
- (rval >> 16) & 0xff,\r
- (rval >> 24) & 0xff);\r
+ printf ("%d Server Net Address: %d.%d.%d.%d port %lld\n", getpid(),\r
+ (rval >> 0) & 0xff, (rval >> 8) & 0xff,\r
+ (rval >> 16) & 0xff, (rval >> 24) & 0xff, conn_id);\r
\r
remote_addr = *((DAT_IA_ADDRESS_PTR)target->ai_addr);\r
+ freeaddrinfo(target);\r
\r
- LOGPRINTF1("%d Connecting to server\n",getpid());\r
+ LOGPRINTF("%d Connecting to server\n",getpid());\r
ret = dat_ep_connect( h_ep,\r
&remote_addr,\r
conn_id,\r
return(ret);\r
}\r
else\r
- LOGPRINTF1("%d dat_ep_connect completed\n", getpid());\r
+ LOGPRINTF("%d dat_ep_connect completed\n", getpid());\r
}\r
\r
printf("%d Waiting for connect response\n",getpid());\r
return(ret);\r
}\r
else\r
- LOGPRINTF1("%d dat_evd_wait for h_conn_evd completed\n", getpid());\r
+ LOGPRINTF("%d dat_evd_wait for h_conn_evd completed\n", getpid());\r
\r
if ( event.event_number != DAT_CONNECTION_EVENT_ESTABLISHED ) {\r
fprintf(stderr, "%d Error unexpected conn event : %s\n",\r
/*\r
* Setup our remote memory and tell the other side about it\r
*/\r
- va = (DAT_VADDR)rbuf;\r
- len = RDMA_BUFFER_SIZE;\r
- rmr_ctx = rmr_context_recv;\r
-\r
- rmr_send_msg.virtual_address = cl_hton64((DAT_VADDR)rbuf);\r
- rmr_send_msg.segment_length = cl_hton32(RDMA_BUFFER_SIZE);\r
- rmr_send_msg.rmr_context = cl_hton32(rmr_context_recv);\r
+ rmr_send_msg.virtual_address = hton64((DAT_VADDR)rbuf);\r
+ rmr_send_msg.segment_length = hton32(RDMA_BUFFER_SIZE);\r
+ rmr_send_msg.rmr_context = hton32(rmr_context_recv);\r
\r
- printf("%d Send RMR to remote: snd_msg: r_key_ctx=%lx,va=%I64x,len=0x%lx\n",\r
- getpid(), rmr_ctx, va, len );\r
+ printf("%d Send RMR msg to remote: r_key_ctx=0x%x,va=%p,len=0x%x\n",\r
+ getpid(), rmr_context_recv, rbuf, RDMA_BUFFER_SIZE );\r
\r
- ret = send_msg( &rmr_send_msg,\r
+ ret = send_msg(&rmr_send_msg,\r
sizeof( DAT_RMR_TRIPLET ),\r
lmr_context_send_msg,\r
cookie,\r
- DAT_COMPLETION_SUPPRESS_FLAG );\r
+ DAT_COMPLETION_SUPPRESS_FLAG);\r
\r
if(ret != DAT_SUCCESS) {\r
fprintf(stderr, "%d Error send_msg: %s\n",\r
return(ret);\r
}\r
else\r
- LOGPRINTF1("%d send_msg completed\n", getpid());\r
+ LOGPRINTF("%d send_msg completed\n", getpid());\r
\r
/*\r
* Wait for remote RMR information for RDMA\r
{\r
DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;\r
ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd );\r
- LOGPRINTF2("%d cno wait return evd_handle=%p\n", getpid(),evd);\r
+ LOGPRINTF("%d cno wait return evd_handle=%p\n", getpid(),evd);\r
if ( evd != h_dto_rcv_evd ) {\r
fprintf(stderr,\r
"%d Error waiting on h_dto_cno: evd != h_dto_rcv_evd\n",\r
return(ret);\r
}\r
else {\r
- LOGPRINTF1("%d dat_evd_wait h_dto_rcv_evd completed\n", getpid());\r
+ LOGPRINTF("%d dat_evd_wait h_dto_rcv_evd completed\n", getpid());\r
}\r
}\r
\r
sizeof( DAT_RMR_TRIPLET )) ||\r
(event.event_data.dto_completion_event_data.user_cookie.as_64 !=\r
recv_msg_index) ) {\r
- fprintf(stderr,"ERR recv event: len=%d cookie= %llx expected %d/%d\n",\r
+ fprintf(stderr,"ERR recv event: len=%d cookie="F64x" expected %d/%d\n",\r
(int)event.event_data.dto_completion_event_data.transfered_length,\r
(int)event.event_data.dto_completion_event_data.user_cookie.as_64,\r
sizeof(DAT_RMR_TRIPLET), recv_msg_index );\r
return( DAT_ABORT );\r
}\r
\r
- rmr_recv_msg[ recv_msg_index ].virtual_address =\r
- cl_ntoh64(rmr_recv_msg[ recv_msg_index ].virtual_address);\r
- rmr_recv_msg[ recv_msg_index ].segment_length =\r
- cl_ntoh32(rmr_recv_msg[ recv_msg_index ].segment_length);\r
- rmr_recv_msg[ recv_msg_index ].rmr_context =\r
- cl_ntoh32(rmr_recv_msg[ recv_msg_index ].rmr_context);\r
-\r
- r_iov = rmr_recv_msg[ recv_msg_index ];\r
-\r
- printf("%d Received RMR from remote: r_iov: ctx=%x,va=%I64x,len=0x%x\n",\r
- getpid(),\r
- r_iov.rmr_context, \r
- r_iov.virtual_address,\r
- r_iov.segment_length );\r
+ /* swap received RMR msg: network order to host order */\r
+ r_iov = rmr_recv_msg[recv_msg_index];\r
+ rmr_recv_msg[recv_msg_index].rmr_context = \r
+ ntoh32(r_iov.rmr_context);\r
+ rmr_recv_msg[recv_msg_index].virtual_address = \r
+ ntoh64(r_iov.virtual_address);\r
+ rmr_recv_msg[recv_msg_index].segment_length = \r
+ ntoh32(r_iov.segment_length);\r
+\r
+ printf("%d Received RMR from remote: "\r
+ "r_iov: r_key_ctx=%x,va="F64x",len=0x%x\n",\r
+ getpid(), rmr_recv_msg[recv_msg_index].rmr_context,\r
+ rmr_recv_msg[recv_msg_index].virtual_address,\r
+ rmr_recv_msg[recv_msg_index].segment_length );\r
\r
recv_msg_index++;\r
\r
return ( DAT_SUCCESS );\r
}\r
\r
-\r
void\r
-disconnect_ep()\r
+disconnect_ep( void )\r
{\r
DAT_RETURN ret;\r
DAT_EVENT event;\r
* disconnect request and then exit.\r
*/\r
if ( !server ) {\r
- LOGPRINTF1("%d dat_ep_disconnect\n", getpid());\r
+ LOGPRINTF("%d dat_ep_disconnect\n", getpid());\r
ret = dat_ep_disconnect( h_ep, DAT_CLOSE_DEFAULT );\r
if(ret != DAT_SUCCESS) {\r
fprintf(stderr, "%d Error dat_ep_disconnect: %s\n",\r
- getpid(),DT_RetToString(ret));\r
+ getpid(),DT_RetToString(ret));\r
}\r
else {\r
- LOGPRINTF1("%d dat_ep_disconnect completed\n", getpid());\r
+ LOGPRINTF("%d dat_ep_disconnect completed\n", getpid());\r
}\r
}\r
else {\r
- LOGPRINTF1("%d Server waiting for disconnect...\n", getpid());\r
+ LOGPRINTF("%d Server waiting for disconnect...\n", getpid());\r
}\r
\r
ret = dat_evd_wait(h_conn_evd, DAT_TIMEOUT_INFINITE, 1, &event, &nmore);\r
getpid(),DT_RetToString(ret));\r
}\r
else {\r
- LOGPRINTF1("%d dat_evd_wait for h_conn_evd completed\n", getpid());\r
+ LOGPRINTF("%d dat_evd_wait for h_conn_evd completed\n", getpid());\r
}\r
}\r
\r
getpid(),DT_RetToString(ret));\r
}\r
else {\r
- LOGPRINTF1("%d dat_psp_free completed\n", getpid());\r
+ LOGPRINTF("%d dat_psp_free completed\n", getpid());\r
}\r
}\r
\r
/* Unregister Send message Buffer */\r
if ( h_lmr_send_msg != DAT_HANDLE_NULL ) {\r
- LOGPRINTF2("%d Unregister send message h_lmr %p \n",getpid(),h_lmr_send_msg);\r
+ LOGPRINTF("%d Unregister send message h_lmr %p \n",getpid(),h_lmr_send_msg);\r
ret = dat_lmr_free(h_lmr_send_msg);\r
if(ret != DAT_SUCCESS) {\r
fprintf(stderr, "%d Error deregistering send msg mr: %s\n",\r
getpid(), DT_RetToString(ret));\r
} else {\r
- LOGPRINTF1("%d Unregistered send message Buffer\n",getpid());\r
+ LOGPRINTF("%d Unregistered send message Buffer\n",getpid());\r
h_lmr_send_msg = NULL;\r
}\r
}\r
\r
/* Unregister recv message Buffer */\r
if ( h_lmr_recv_msg != DAT_HANDLE_NULL ) {\r
- LOGPRINTF2("%d Unregister recv message h_lmr %p \n",getpid(),h_lmr_recv_msg);\r
+ LOGPRINTF("%d Unregister recv message h_lmr %p \n",getpid(),h_lmr_recv_msg);\r
ret = dat_lmr_free(h_lmr_recv_msg);\r
if(ret != DAT_SUCCESS) {\r
fprintf(stderr, "%d Error deregistering recv msg mr: %s\n",\r
getpid(), DT_RetToString(ret));\r
} else {\r
- LOGPRINTF1("%d Unregistered recv message Buffer\n",getpid());\r
+ LOGPRINTF("%d Unregistered recv message Buffer\n",getpid());\r
h_lmr_recv_msg = NULL;\r
}\r
}\r
\r
\r
DAT_RETURN\r
-do_rdma_write_with_msg( )\r
+do_rdma_write_with_msg( void )\r
{\r
DAT_EVENT event;\r
DAT_COUNT nmore;\r
l_iov[i].virtual_address = (DAT_VADDR)\r
(&sbuf[l_iov[i].segment_length*i]);\r
\r
- LOGPRINTF4("%d rdma_write iov[%d] buf=%p,len=%d\n", \r
+ LOGPRINTF("%d rdma_write iov[%d] buf=%p,len=%d\n", \r
getpid(), i, &sbuf[l_iov[i].segment_length*i],\r
l_iov[i].segment_length);\r
}\r
getpid(),DT_RetToString(ret));\r
return( DAT_ABORT );\r
}\r
- LOGPRINTF2("%d rdma_write # %d completed\n", getpid(),i+1);\r
+ LOGPRINTF("%d rdma_write # %d completed\n", getpid(),i+1);\r
}\r
\r
/*\r
* Send RMR information a 2nd time to indicate completion\r
+ * NOTE: already swapped to network order in connect_ep\r
*/\r
- rmr_send_msg.virtual_address = cl_ntoh64((DAT_VADDR)rbuf);\r
- rmr_send_msg.segment_length = cl_ntoh32(RDMA_BUFFER_SIZE);\r
- rmr_send_msg.rmr_context = cl_hton32(rmr_context_recv);\r
-\r
- printf("%d Sending completion message\n",getpid());\r
+ printf("%d Sending RDMA WRITE completion message\n",getpid());\r
\r
ret = send_msg( &rmr_send_msg,\r
sizeof( DAT_RMR_TRIPLET ),\r
getpid(),DT_RetToString(ret));\r
return(ret);\r
} else {\r
- LOGPRINTF1("%d send_msg completed\n", getpid());\r
+ LOGPRINTF("%d send_msg completed\n", getpid());\r
}\r
\r
/*\r
- * Collect first event, write completion or the inbound recv with immed\r
+ * Collect first event, write completion or the inbound recv \r
*/\r
- if ( polling ) {\r
- while ( dat_evd_dequeue( h_dto_rcv_evd, &event ) == DAT_QUEUE_EMPTY )\r
+ if (polling) {\r
+ while (dat_evd_dequeue(h_dto_rcv_evd, &event) == DAT_QUEUE_EMPTY)\r
rdma_wr_poll_count++;\r
}\r
else {\r
- LOGPRINTF1("%d waiting for message receive event\n", getpid());\r
+ LOGPRINTF("%d waiting for message receive event\n", getpid());\r
if (use_cno) {\r
DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;\r
ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd );\r
- LOGPRINTF2("%d cno wait return evd_handle=%p\n",\r
+ LOGPRINTF("%d cno wait return evd_handle=%p\n",\r
getpid(),evd);\r
if ( evd != h_dto_rcv_evd ) {\r
fprintf(stderr, "%d Error waiting on h_dto_cno: "\r
}\r
}\r
/* use wait to dequeue */\r
- ret = dat_evd_wait( h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore );\r
+ ret = dat_evd_wait(h_dto_rcv_evd, DTO_TIMEOUT, 1, &event, &nmore);\r
if (ret != DAT_SUCCESS) {\r
fprintf(stderr, "%d: ERROR: DTO dat_evd_wait() %s\n",\r
getpid(),DT_RetToString(ret));\r
\r
if ( (event.event_data.dto_completion_event_data.transfered_length != sizeof( DAT_RMR_TRIPLET )) ||\r
(event.event_data.dto_completion_event_data.user_cookie.as_64 != recv_msg_index) ) {\r
- fprintf(stderr,"unexpected event data for receive: len=%d cookie=" PRIx64 " exp %d/%d\n",\r
+ fprintf(stderr,"unexpected event data for receive: len=%d cookie="F64x" exp %d/%d\n",\r
(int)event.event_data.dto_completion_event_data.transfered_length,\r
(int)event.event_data.dto_completion_event_data.user_cookie.as_64,\r
sizeof(DAT_RMR_TRIPLET), recv_msg_index );\r
return( DAT_ABORT );\r
}\r
\r
- rmr_recv_msg[ recv_msg_index ].virtual_address =\r
- cl_ntoh64(rmr_recv_msg[ recv_msg_index ].virtual_address);\r
- rmr_recv_msg[ recv_msg_index ].segment_length =\r
- cl_ntoh32(rmr_recv_msg[ recv_msg_index ].segment_length);\r
- rmr_recv_msg[ recv_msg_index ].rmr_context =\r
- cl_ntoh32(rmr_recv_msg[ recv_msg_index ].rmr_context);\r
+ /* swap received RMR msg: network order to host order */\r
+ r_iov = rmr_recv_msg[recv_msg_index];\r
+ rmr_recv_msg[recv_msg_index].virtual_address =\r
+ ntoh64(rmr_recv_msg[recv_msg_index].virtual_address);\r
+ rmr_recv_msg[recv_msg_index].segment_length =\r
+ ntoh32(rmr_recv_msg[recv_msg_index].segment_length);\r
+ rmr_recv_msg[recv_msg_index].rmr_context =\r
+ ntoh32(rmr_recv_msg[recv_msg_index].rmr_context);\r
\r
- r_iov = rmr_recv_msg[ recv_msg_index ];\r
+ printf("%d Received RMR from remote: "\r
+ "r_iov: r_key_ctx=%x,va="F64x",len=0x%x\n",\r
+ getpid(), rmr_recv_msg[recv_msg_index].rmr_context,\r
+ rmr_recv_msg[recv_msg_index].virtual_address,\r
+ rmr_recv_msg[recv_msg_index].segment_length);\r
\r
- printf("%d Received RMR from remote: r_iov: ctx=%x,va=%I64x,len=0x%x\n",\r
- getpid(), r_iov.rmr_context,\r
- (void*)r_iov.virtual_address,\r
- r_iov.segment_length );\r
-\r
- LOGPRINTF1("%d inbound rdma_write; send msg event SUCCESS!!\n",getpid());\r
+ LOGPRINTF("%d inbound rdma_write; send msg event SUCCESS!!\n",\r
+ getpid());\r
\r
printf("%d %s RDMA write buffer contains: %s\n",\r
getpid(),\r
}\r
\r
DAT_RETURN\r
-do_rdma_read_with_msg( )\r
+do_rdma_read_with_msg( void )\r
{\r
DAT_EVENT event;\r
DAT_COUNT nmore;\r
\r
printf("\n %d RDMA READ DATA with SEND MSG\n\n",getpid());\r
\r
- if ( recv_msg_index >= MSG_BUF_COUNT )\r
+ if (recv_msg_index >= MSG_BUF_COUNT)\r
return( DAT_ABORT );\r
\r
/* get RMR information from previously received message */\r
- r_iov = rmr_recv_msg[ recv_msg_index-1 ];\r
+ r_iov = rmr_recv_msg[recv_msg_index-1];\r
\r
/* setup rdma read buffer to initial string to be overwritten */\r
- strcpy( (char*)sbuf, "blah, blah, blah\n" );\r
+ strcpy((char*)sbuf, "blah, blah, blah\n");\r
\r
- if ( server )\r
- strcpy( (char*)rbuf, "server RDMA read data..." );\r
+ if (server)\r
+ strcpy((char*)rbuf, "server RDMA read data...");\r
else\r
- strcpy( (char*)rbuf, "client RDMA read data..." );\r
+ strcpy((char*)rbuf, "client RDMA read data...");\r
\r
l_iov.lmr_context = lmr_context_send;\r
l_iov.virtual_address = (DAT_VADDR)sbuf;\r
for (i=0;i<MAX_RDMA_RD;i++) {\r
cookie.as_64 = 0x9999;\r
start = get_time();\r
- ret = dat_ep_post_rdma_read( h_ep, // ep_handle\r
- 1, // num_segments\r
- &l_iov, // LMR\r
- cookie, // user_cookie\r
- &r_iov, // RMR\r
- DAT_COMPLETION_DEFAULT_FLAG );\r
+ ret = dat_ep_post_rdma_read(h_ep, // ep_handle\r
+ 1, // num_segments\r
+ &l_iov, // LMR\r
+ cookie, // user_cookie\r
+ &r_iov, // RMR\r
+ DAT_COMPLETION_DEFAULT_FLAG );\r
if (ret != DAT_SUCCESS) {\r
fprintf(stderr, "%d: ERROR: dat_ep_post_rdma_read() %s\n", \r
getpid(),DT_RetToString(ret));\r
- return( DAT_ABORT );\r
+ return(DAT_ABORT);\r
}\r
\r
if (polling) {\r
rdma_rd_poll_count[i]++;\r
} \r
else {\r
- LOGPRINTF1("%d waiting for rdma_read completion event\n", getpid());\r
+ LOGPRINTF("%d waiting for rdma_read completion event\n", getpid());\r
if (use_cno) {\r
DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;\r
ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd );\r
- LOGPRINTF2("%d cno wait return evd_handle=%p\n", getpid(),evd);\r
+ LOGPRINTF("%d cno wait return evd_handle=%p\n", getpid(),evd);\r
if ( evd != h_dto_req_evd ) {\r
fprintf(stderr, \r
"%d Error waiting on h_dto_cno: evd != h_dto_req_evd\n", \r
}\r
if ((event.event_data.dto_completion_event_data.transfered_length != buf_len ) ||\r
(event.event_data.dto_completion_event_data.user_cookie.as_64 != 0x9999 )) {\r
- fprintf(stderr, "%d: ERROR: DTO len %d or cookie " PRIx64 "\n", \r
+ fprintf(stderr, "%d: ERROR: DTO len %d or cookie "F64x"\n", \r
getpid(),\r
event.event_data.dto_completion_event_data.transfered_length,\r
event.event_data.dto_completion_event_data.user_cookie.as_64 );\r
time.rdma_rd[i] = ((stop - start)*1.0e6);\r
time.rdma_rd_total += time.rdma_rd[i];\r
\r
- LOGPRINTF2("%d rdma_read # %d completed\n", getpid(),i+1);\r
+ LOGPRINTF("%d rdma_read # %d completed\n", getpid(),i+1);\r
}\r
\r
/*\r
- * Send RMR information a 2nd time to indicate completion\r
+ * Send RMR information a 3rd time to indicate completion\r
+ * NOTE: already swapped to network order in connect_ep\r
*/\r
- rmr_send_msg.virtual_address = cl_hton64((DAT_VADDR)rbuf);\r
- rmr_send_msg.segment_length = cl_hton32(RDMA_BUFFER_SIZE);\r
- rmr_send_msg.rmr_context = cl_hton32(rmr_context_recv);\r
-\r
- printf("%d Sending completion message\n",getpid());\r
+ printf("%d Sending RDMA read completion message\n",getpid());\r
\r
ret = send_msg( &rmr_send_msg,\r
sizeof( DAT_RMR_TRIPLET ),\r
getpid(),DT_RetToString(ret));\r
return(ret);\r
} else {\r
- LOGPRINTF1("%d send_msg completed\n", getpid());\r
+ LOGPRINTF("%d send_msg completed\n", getpid());\r
}\r
\r
/*\r
while ( dat_evd_dequeue( h_dto_rcv_evd, &event ) == DAT_QUEUE_EMPTY );\r
}\r
else {\r
- LOGPRINTF1("%d waiting for message receive event\n", getpid());\r
+ LOGPRINTF("%d waiting for message receive event\n", getpid());\r
if (use_cno) {\r
DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;\r
ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd );\r
- LOGPRINTF2("%d cno wait return evd_handle=%p\n", getpid(),evd);\r
+ LOGPRINTF("%d cno wait return evd_handle=%p\n", getpid(),evd);\r
if ( evd != h_dto_rcv_evd ) {\r
fprintf(stderr, \r
"%d Error waiting on h_dto_cno: evd != h_dto_rcv_evd\n",\r
\r
/* validate event number and status */\r
printf("%d inbound rdma_read; send message arrived!\n",getpid());\r
- if ( event.event_number != DAT_DTO_COMPLETION_EVENT ) {\r
+ if (event.event_number != DAT_DTO_COMPLETION_EVENT ) {\r
fprintf(stderr, "%d Error unexpected DTO event : %s\n",\r
getpid(),DT_EventToSTr(event.event_number));\r
return( DAT_ABORT );\r
}\r
\r
- if ( (event.event_data.dto_completion_event_data.transfered_length != sizeof( DAT_RMR_TRIPLET )) ||\r
- (event.event_data.dto_completion_event_data.user_cookie.as_64 != recv_msg_index) ) {\r
+ if ((event.event_data.dto_completion_event_data.transfered_length != sizeof( DAT_RMR_TRIPLET)) ||\r
+ (event.event_data.dto_completion_event_data.user_cookie.as_64 != recv_msg_index)) {\r
\r
- fprintf(stderr,"unexpected event data for receive: len=%d cookie=" PRIx64 " exp %d/%d\n",\r
+ fprintf(stderr,"unexpected event data for receive: len=%d cookie="F64x" exp %d/%d\n",\r
(int)event.event_data.dto_completion_event_data.transfered_length,\r
(int)event.event_data.dto_completion_event_data.user_cookie.as_64,\r
sizeof(DAT_RMR_TRIPLET), recv_msg_index );\r
return( DAT_ABORT );\r
}\r
\r
- rmr_recv_msg[ recv_msg_index ].virtual_address =\r
- cl_ntoh64(rmr_recv_msg[ recv_msg_index ].virtual_address);\r
- rmr_recv_msg[ recv_msg_index ].segment_length =\r
- cl_ntoh32(rmr_recv_msg[ recv_msg_index ].segment_length);\r
- rmr_recv_msg[ recv_msg_index ].rmr_context =\r
- cl_ntoh32(rmr_recv_msg[ recv_msg_index ].rmr_context);\r
-\r
- r_iov = rmr_recv_msg[ recv_msg_index ];\r
+ /* swap received RMR msg: network order to host order */\r
+ r_iov = rmr_recv_msg[recv_msg_index];\r
+ rmr_recv_msg[recv_msg_index].virtual_address =\r
+ ntoh64(rmr_recv_msg[recv_msg_index].virtual_address);\r
+ rmr_recv_msg[recv_msg_index].segment_length =\r
+ ntoh32(rmr_recv_msg[ recv_msg_index].segment_length);\r
+ rmr_recv_msg[recv_msg_index].rmr_context =\r
+ ntoh32(rmr_recv_msg[recv_msg_index].rmr_context);\r
\r
- printf("%d Received RMR from remote: r_iov: ctx=%x,va=%I64x,len=0x%x\n",\r
- getpid(), (r_iov.rmr_context), \r
- (void*)r_iov.virtual_address, r_iov.segment_length );\r
+ printf("%d Received RMR from remote: "\r
+ "r_iov: r_key_ctx=%x,va="F64x",len=0x%x\n",\r
+ getpid(), rmr_recv_msg[recv_msg_index].rmr_context,\r
+ rmr_recv_msg[recv_msg_index].virtual_address,\r
+ rmr_recv_msg[recv_msg_index].segment_length);\r
\r
- LOGPRINTF1("%d inbound rdma_write; send msg event SUCCESS!!\n",getpid());\r
+ LOGPRINTF("%d inbound rdma_write; send msg event SUCCESS!!\n",getpid());\r
\r
printf("%d %s RCV RDMA read buffer contains: %s\n",\r
getpid(),\r
l_iov.virtual_address = (DAT_VADDR)rcv_buf;\r
l_iov.segment_length = buf_len;\r
\r
- LOGPRINTF2("%d Pre-posting Receive Message Buffers %p\n",\r
+ LOGPRINTF("%d Pre-posting Receive Message Buffers %p\n",\r
getpid(), rcv_buf );\r
\r
ret = dat_ep_post_recv( h_ep,\r
return(ret);\r
}\r
else {\r
- LOGPRINTF2("%d Posted Receive Message Buffer %p\n",getpid(),rcv_buf);\r
+ LOGPRINTF("%d Posted Receive Message Buffer %p\n",getpid(),rcv_buf);\r
}\r
\r
/* next buffer */\r
if ( !server ) {\r
*snd_buf = 0x55;\r
\r
- LOGPRINTF5("%d %s SND buffer %p contains: 0x%x len=%d\n",\r
+ LOGPRINTF("%d %s SND buffer %p contains: 0x%x len=%d\n",\r
getpid(), server ? "SERVER:" : "CLIENT:",\r
snd_buf, *snd_buf, buf_len );\r
\r
return(ret);\r
}\r
else {\r
- LOGPRINTF1("%d send_msg completed\n", getpid());\r
+ LOGPRINTF("%d send_msg completed\n", getpid());\r
}\r
}\r
\r
/* Wait for recv message */\r
if ( polling ) {\r
poll_count=0;\r
- LOGPRINTF1("%d Polling for message receive event\n", getpid());\r
+ LOGPRINTF("%d Polling for message receive event\n", getpid());\r
while ( dat_evd_dequeue( h_dto_rcv_evd, &event ) == DAT_QUEUE_EMPTY )\r
poll_count++;\r
}\r
else {\r
- LOGPRINTF1("%d waiting for message receive event\n", getpid());\r
+ LOGPRINTF("%d waiting for message receive event\n", getpid());\r
if (use_cno) {\r
DAT_EVD_HANDLE evd = DAT_HANDLE_NULL;\r
ret = dat_cno_wait( h_dto_cno, DTO_TIMEOUT, &evd );\r
- LOGPRINTF2("%d cno wait return evd_handle=%p\n", getpid(),evd);\r
+ LOGPRINTF("%d cno wait return evd_handle=%p\n", getpid(),evd);\r
if ( evd != h_dto_rcv_evd )\r
{\r
fprintf(stderr, \r
start = get_time();\r
}\r
/* validate event number and status */\r
- LOGPRINTF1("%d inbound message; message arrived!\n",getpid());\r
+ LOGPRINTF("%d inbound message; message arrived!\n",getpid());\r
if ( event.event_number != DAT_DTO_COMPLETION_EVENT ) {\r
fprintf(stderr, "%d Error unexpected DTO event : %s\n",\r
getpid(),DT_EventToSTr(event.event_number));\r
!= buf_len) ||\r
(event.event_data.dto_completion_event_data.user_cookie.as_64\r
!= burst_msg_index) ) {\r
- fprintf(stderr,"ERR: recv event: len=%d cookie=" PRIx64 " exp %d/%d\n",\r
+ fprintf(stderr,"ERR: recv event: len=%d cookie="F64x" exp %d/%d\n",\r
(int)event.event_data.dto_completion_event_data.transfered_length,\r
(int)event.event_data.dto_completion_event_data.user_cookie.as_64,\r
buf_len, burst_msg_index );\r
return( DAT_ABORT );\r
}\r
\r
- LOGPRINTF5("%d %s RCV buffer %p contains: 0x%x len=%d\n",\r
+ LOGPRINTF("%d %s RCV buffer %p contains: 0x%x len=%d\n",\r
getpid(), server ? "SERVER:" : "CLIENT:",\r
rcv_buf, *rcv_buf, buf_len );\r
\r
if ( server ) {\r
*snd_buf = 0xaa;\r
\r
- LOGPRINTF5("%d %s SND buffer %p contains: 0x%x len=%d\n",\r
+ LOGPRINTF("%d %s SND buffer %p contains: 0x%x len=%d\n",\r
getpid(), server ? "SERVER:" : "CLIENT:",\r
snd_buf, *snd_buf, buf_len );\r
\r
return(ret);\r
}\r
else {\r
- LOGPRINTF1("%d send_msg completed\n", getpid());\r
+ LOGPRINTF("%d send_msg completed\n", getpid());\r
}\r
}\r
\r
getpid(),DT_RetToString(ret));\r
return (ret);\r
} else {\r
- LOGPRINTF2("%d Registered Receive RDMA Buffer %p\n",\r
+ LOGPRINTF("%d Registered Receive RDMA Buffer %p\n",\r
getpid(),region.for_va);\r
}\r
\r
getpid(),DT_RetToString(ret));\r
return (ret);\r
} else {\r
- LOGPRINTF2("%d Registered Send RDMA Buffer %p\n",\r
+ LOGPRINTF("%d Registered Send RDMA Buffer %p\n",\r
getpid(),region.for_va);\r
}\r
\r
/* Unregister Recv Buffer */\r
if ( h_lmr_recv != DAT_HANDLE_NULL )\r
{\r
- LOGPRINTF2("%d Unregister h_lmr %p \n",getpid(),h_lmr_recv);\r
+ LOGPRINTF("%d Unregister h_lmr %p \n",getpid(),h_lmr_recv);\r
start = get_time();\r
ret = dat_lmr_free(h_lmr_recv);\r
stop = get_time();\r
return (ret);\r
}\r
else {\r
- LOGPRINTF1("%d Unregistered Recv Buffer\n",getpid());\r
+ LOGPRINTF("%d Unregistered Recv Buffer\n",getpid());\r
h_lmr_recv = NULL;\r
}\r
}\r
\r
/* Unregister Send Buffer */\r
if ( h_lmr_send != DAT_HANDLE_NULL ) {\r
- LOGPRINTF2("%d Unregister h_lmr %p \n",getpid(),h_lmr_send);\r
+ LOGPRINTF("%d Unregister h_lmr %p \n",getpid(),h_lmr_send);\r
ret = dat_lmr_free(h_lmr_send);\r
if(ret != DAT_SUCCESS) {\r
fprintf(stderr, "%d Error deregistering send mr: %s\n",\r
return (ret);\r
}\r
else {\r
- LOGPRINTF1("%d Unregistered send Buffer\n",getpid());\r
+ LOGPRINTF("%d Unregistered send Buffer\n",getpid());\r
h_lmr_send = NULL;\r
}\r
}\r
create_events(void)\r
{\r
DAT_RETURN ret;\r
+ DAT_EVD_PARAM param;\r
\r
/* create CNO */\r
if (use_cno) {\r
return (ret);\r
}\r
else {\r
- LOGPRINTF2("%d cr_evd created, %p\n", getpid(), h_dto_cno);\r
+ LOGPRINTF("%d cr_evd created, %p\n", getpid(), h_dto_cno);\r
}\r
}\r
\r
return (ret);\r
}\r
else {\r
- LOGPRINTF2("%d cr_evd created %p\n", getpid(),h_cr_evd);\r
+ LOGPRINTF("%d cr_evd created %p\n", getpid(),h_cr_evd);\r
}\r
\r
/* create conn EVD */\r
return (ret);\r
}\r
else {\r
- LOGPRINTF2("%d con_evd created %p\n", getpid(),h_conn_evd);\r
+ LOGPRINTF("%d con_evd created %p\n", getpid(),h_conn_evd);\r
}\r
\r
/* create dto SND EVD, with CNO if use_cno was set */\r
ret = dat_evd_create( h_ia,\r
- MSG_BUF_COUNT+MAX_RDMA_RD+burst*2,\r
+ (MSG_BUF_COUNT+MAX_RDMA_RD+burst)*2,\r
h_dto_cno,\r
DAT_EVD_DTO_FLAG,\r
&h_dto_req_evd );\r
return (ret);\r
}\r
else {\r
- LOGPRINTF2("%d dto_req_evd created %p\n", getpid(), h_dto_req_evd );\r
+ LOGPRINTF("%d dto_req_evd created %p\n", getpid(), h_dto_req_evd );\r
}\r
\r
/* create dto RCV EVD, with CNO if use_cno was set */\r
return (ret);\r
}\r
else {\r
- LOGPRINTF2("%d dto_rcv_evd created %p\n", getpid(), h_dto_rcv_evd );\r
+ LOGPRINTF("%d dto_rcv_evd created %p\n", getpid(), h_dto_rcv_evd );\r
+ }\r
+\r
+ /* query DTO req EVD and check size */\r
+ ret = dat_evd_query( h_dto_req_evd,\r
+ DAT_EVD_FIELD_EVD_QLEN,\r
+ ¶m );\r
+ if(ret != DAT_SUCCESS) {\r
+ fprintf(stderr, "%d Error dat_evd_query request evd: %s\n",\r
+ getpid(),DT_RetToString(ret));\r
+ return (ret);\r
+ }\r
+ else if (param.evd_qlen < (MSG_BUF_COUNT+MAX_RDMA_RD+burst)*2) {\r
+ fprintf(stderr, "%d Error dat_evd qsize too small: %d < %d\n",\r
+ getpid(), param.evd_qlen, \r
+ (MSG_BUF_COUNT+MAX_RDMA_RD+burst)*2);\r
+ return (ret);\r
}\r
+ \r
+ LOGPRINTF("%d dto_req_evd QLEN - requested %d and actual %d\n", \r
+ getpid(), (MSG_BUF_COUNT+MAX_RDMA_RD+burst)*2,\r
+ param.evd_qlen);\r
\r
return DAT_SUCCESS;\r
}\r
\r
/* free cr EVD */\r
if ( h_cr_evd != DAT_HANDLE_NULL ) {\r
- LOGPRINTF2("%d Free cr EVD %p \n",getpid(),h_cr_evd);\r
+ LOGPRINTF("%d Free cr EVD %p \n",getpid(),h_cr_evd);\r
ret = dat_evd_free( h_cr_evd );\r
if(ret != DAT_SUCCESS) {\r
fprintf(stderr, "%d Error freeing cr EVD: %s\n",\r
getpid(), DT_RetToString(ret));\r
return (ret);\r
} else {\r
- LOGPRINTF1("%d Freed cr EVD\n",getpid());\r
+ LOGPRINTF("%d Freed cr EVD\n",getpid());\r
h_cr_evd = DAT_HANDLE_NULL;\r
}\r
}\r
\r
/* free conn EVD */\r
if ( h_conn_evd != DAT_HANDLE_NULL ) {\r
- LOGPRINTF2("%d Free conn EVD %p \n",getpid(),h_conn_evd);\r
+ LOGPRINTF("%d Free conn EVD %p \n",getpid(),h_conn_evd);\r
ret = dat_evd_free( h_conn_evd );\r
if(ret != DAT_SUCCESS) {\r
fprintf(stderr, "%d Error freeing conn EVD: %s\n",\r
return (ret);\r
}\r
else {\r
- LOGPRINTF1("%d Freed conn EVD\n",getpid());\r
+ LOGPRINTF("%d Freed conn EVD\n",getpid());\r
h_conn_evd = DAT_HANDLE_NULL;\r
}\r
}\r
\r
/* free RCV dto EVD */\r
if ( h_dto_rcv_evd != DAT_HANDLE_NULL ) {\r
- LOGPRINTF2("%d Free RCV dto EVD %p \n",getpid(),h_dto_rcv_evd);\r
+ LOGPRINTF("%d Free RCV dto EVD %p \n",getpid(),h_dto_rcv_evd);\r
start = get_time();\r
ret = dat_evd_free( h_dto_rcv_evd );\r
stop = get_time();\r
return (ret);\r
}\r
else {\r
- LOGPRINTF1("%d Freed dto EVD\n",getpid());\r
+ LOGPRINTF("%d Freed dto EVD\n",getpid());\r
h_dto_rcv_evd = DAT_HANDLE_NULL;\r
}\r
}\r
\r
/* free REQ dto EVD */\r
if ( h_dto_req_evd != DAT_HANDLE_NULL ) {\r
- LOGPRINTF2("%d Free REQ dto EVD %p \n",getpid(),h_dto_req_evd);\r
+ LOGPRINTF("%d Free REQ dto EVD %p \n",getpid(),h_dto_req_evd);\r
ret = dat_evd_free( h_dto_req_evd );\r
if(ret != DAT_SUCCESS) {\r
fprintf(stderr, "%d Error freeing dto EVD: %s\n",\r
return (ret);\r
}\r
else {\r
- LOGPRINTF1("%d Freed dto EVD\n",getpid());\r
+ LOGPRINTF("%d Freed dto EVD\n",getpid());\r
h_dto_req_evd = DAT_HANDLE_NULL;\r
}\r
}\r
\r
/* free CNO */\r
if ( h_dto_cno != DAT_HANDLE_NULL ) {\r
- LOGPRINTF2("%d Free dto CNO %p \n",getpid(),h_dto_cno);\r
+ LOGPRINTF("%d Free dto CNO %p \n",getpid(),h_dto_cno);\r
start = get_time();\r
ret = dat_cno_free( h_dto_cno );\r
stop = get_time();\r
return (ret);\r
}\r
else {\r
- LOGPRINTF1("%d Freed dto CNO\n",getpid());\r
+ LOGPRINTF("%d Freed dto CNO\n",getpid());\r
h_dto_cno = DAT_HANDLE_NULL;\r
}\r
}\r
{\r
printf("\n DAPL USAGE \n\n");\r
printf("s: server\n");\r
+ printf("t: performance times\n");\r
printf("c: use cno\n");\r
printf("v: verbose\n");\r
printf("p: polling\n");\r
printf("d: delay before accept\n");\r
printf("b: buf length to allocate\n");\r
printf("B: burst count, rdma and msgs \n");\r
- printf("h: hostname\n");\r
+ printf("h: hostname/address of server, specified on client\n");\r
printf("P: provider name (default = OpenIB-cma)\n");\r
printf("\n");\r
}\r
/*\r
- * Copyright (c) 2007 Intel Corporation. All rights reserved.\r
+ * Copyright (c) 2007-2008 Intel Corporation. All rights reserved.\r
*\r
* This software is available to you under a choice of one of two\r
* licenses. You may choose to be licensed under the terms of the GNU\r
#include <ws2tcpip.h>\r
#include <io.h>\r
#include <process.h>\r
+#include <complib/cl_types.h>\r
+#define __BYTE_ORDER __LITTLE_ENDIAN\r
\r
#define getpid _getpid\r
-#define PRIx64 "%llx" \r
-#define DAPL_PROVIDER "ibnic0v2d"\r
+#define F64x "%I64x"\r
+#define DAPL_PROVIDER "ibnic0v2"\r
#else\r
#include <netdb.h>\r
#include <sys/socket.h>\r
#include <netinet/in.h>\r
#include <netinet/tcp.h>\r
#include <arpa/inet.h>\r
-#include <endian.h>\r
-#include <byteswap.h>\r
-#define DAPL_PROVIDER "OpenIB-cma"\r
+#include <inttypes.h>\r
+\r
+#define DAPL_PROVIDER "ofa-v2-ib0"\r
+#define F64x "%"PRIx64""\r
+\r
#endif\r
\r
#include "dat/udat.h"\r
int disconnect_ep(void);\r
\r
#define _OK(status, str) \\r
-{ \\r
- const char *maj_msg, *min_msg; \\r
- if (status != DAT_SUCCESS) { \\r
- dat_strerror(status, &maj_msg, &min_msg); \\r
- fprintf(stderr, str " returned %s : %s\n", maj_msg, min_msg); \\r
- exit(1); \\r
- } \\r
+{\\r
+ const char *maj_msg, *min_msg;\\r
+ if (status != DAT_SUCCESS) {\\r
+ dat_strerror(status, &maj_msg, &min_msg);\\r
+ fprintf(stderr, str " returned %s : %s\n", maj_msg, min_msg);\\r
+ exit(1);\\r
+ }\\r
}\r
\r
-#define _OK2(status, str) \\r
-{ \\r
- const char *maj_msg, *min_msg; \\r
- if (status != DAT_SUCCESS) { \\r
- dat_strerror(status, &maj_msg, &min_msg); \\r
- fprintf(stderr, str " returned %s : %s\n", maj_msg, min_msg); \\r
- exit(1); \\r
- } \\r
+#define _OK2(status, str)\\r
+{\\r
+ const char *maj_msg, *min_msg;\\r
+ if (status != DAT_SUCCESS) {\\r
+ dat_strerror(status, &maj_msg, &min_msg);\\r
+ fprintf(stderr, str " returned %s : %s\n", maj_msg, min_msg);\\r
+ exit(1);\\r
+ }\\r
}\r
\r
+/* byte swap helpers from Complib */\r
+#if __BYTE_ORDER == __LITTLE_ENDIAN\r
+#define ntoh16(x) (uint16_t)( \\r
+ (((uint16_t)(x) & 0x00FF) << 8) | \\r
+ (((uint16_t)(x) & 0xFF00) >> 8))\r
+#define hton16(x) ntoh16(x)\r
+#define ntoh32(x) (uint32_t)( \\r
+ (((uint32_t)(x) & 0x000000FF) << 24)| \\r
+ (((uint32_t)(x) & 0x0000FF00) << 8) | \\r
+ (((uint32_t)(x) & 0x00FF0000) >> 8) | \\r
+ (((uint32_t)(x) & 0xFF000000) >> 24))\r
+#define hton32(x) ntoh32(x)\r
+#define ntoh64(x) (uint64_t)( \\r
+ (((uint64_t)x & 0x00000000000000FF) << 56) | \\r
+ (((uint64_t)x & 0x000000000000FF00) << 40) | \\r
+ (((uint64_t)x & 0x0000000000FF0000) << 24) | \\r
+ (((uint64_t)x & 0x00000000FF000000) << 8 ) | \\r
+ (((uint64_t)x & 0x000000FF00000000) >> 8 ) | \\r
+ (((uint64_t)x & 0x0000FF0000000000) >> 24) | \\r
+ (((uint64_t)x & 0x00FF000000000000) >> 40) | \\r
+ (((uint64_t)x & 0xFF00000000000000) >> 56))\r
+#define hton64(x) ntoh64(x)\r
+#elif __BYTE_ORDER == __BIG_ENDIAN\r
+#define hton16(x) (x)\r
+#define ntoh16(x) (x)\r
+#define hton32(x) (x)\r
+#define ntoh32(x) (x)\r
+#define hton64(x) (x)\r
+#define ntoh64(x) (x)\r
+#endif /* __BYTE_ORDER == __BIG_ENDIAN */\r
+\r
#define DTO_TIMEOUT (1000*1000*5)\r
#define CONN_TIMEOUT (1000*1000*10)\r
#define SERVER_TIMEOUT (1000*1000*120)\r
_OK(status, "dat_ep_post_send");\r
\r
if (! (flags & DAT_COMPLETION_SUPPRESS_FLAG)) {\r
- status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);\r
+ status = dat_evd_wait(dto_evd, DTO_TIMEOUT, \r
+ 1, &event, &nmore);\r
_OK(status, "dat_evd_wait after dat_ep_post_send");\r
\r
if (event.event_number != DAT_DTO_COMPLETION_EVENT) {\r
DAT_EVENT event;\r
DAT_COUNT nmore;\r
DAT_LMR_TRIPLET iov;\r
- DAT_RMR_TRIPLET r_iov;\r
+ DAT_RMR_TRIPLET *r_iov;\r
DAT_DTO_COOKIE cookie;\r
int i;\r
+ DAT_CR_ARRIVAL_EVENT_DATA *cr_event = \r
+ &event.event_data.cr_arrival_event_data;\r
DAT_DTO_COMPLETION_EVENT_DATA *dto_event = \r
&event.event_data.dto_completion_event_data;\r
\r
ep_attr.ep_provider_specific_count = 0;\r
ep_attr.ep_provider_specific = NULL;\r
\r
- status = dat_ep_create(ia, pz, dto_evd, dto_evd, con_evd, &ep_attr, &ep);\r
+ status = dat_ep_create(ia, pz, dto_evd, dto_evd, \r
+ con_evd, &ep_attr, &ep);\r
_OK(status, "dat_ep_create");\r
\r
for (i = 0; i < REG_MEM_COUNT; i++) {\r
region,\r
BUF_SIZE,\r
pz,\r
- DAT_MEM_PRIV_ALL_FLAG|DAT_IB_MEM_PRIV_REMOTE_ATOMIC,\r
+ DAT_MEM_PRIV_ALL_FLAG|\r
+ DAT_IB_MEM_PRIV_REMOTE_ATOMIC,\r
DAT_VA_TYPE_VA,\r
&lmr[ i ],\r
&lmr_context[ i ],\r
region,\r
BUF_SIZE_ATOMIC,\r
pz,\r
- DAT_MEM_PRIV_ALL_FLAG|DAT_IB_MEM_PRIV_REMOTE_ATOMIC,\r
+ DAT_MEM_PRIV_ALL_FLAG|\r
+ DAT_IB_MEM_PRIV_REMOTE_ATOMIC,\r
DAT_VA_TYPE_VA,\r
&lmr_atomic,\r
&lmr_atomic_context,\r
strcpy((char*)buf[ RCV_RDMA_BUF_INDEX ], "blah, blah, blah\n");\r
\r
if (server) {\r
-\r
- strcpy((char*)buf[ SND_RDMA_BUF_INDEX ], "server written data");\r
-\r
+ strcpy((char*)buf[ SND_RDMA_BUF_INDEX ],"server written data");\r
status = dat_psp_create(ia,\r
SERVER_CONN_QUAL,\r
cr_evd,\r
_OK(status, "dat_psp_create");\r
\r
printf("Server waiting for connect request\n");\r
- status = dat_evd_wait(cr_evd, SERVER_TIMEOUT, 1, &event, &nmore);\r
+ status = dat_evd_wait(cr_evd, SERVER_TIMEOUT, \r
+ 1, &event, &nmore);\r
_OK(status, "listen dat_evd_wait");\r
\r
if (event.event_number != DAT_CONNECTION_REQUEST_EVENT) {\r
exit(1);\r
}\r
\r
- if ((event.event_data.cr_arrival_event_data.conn_qual != SERVER_CONN_QUAL) ||\r
- (event.event_data.cr_arrival_event_data.sp_handle.psp_handle != psp)) {\r
-\r
+ if ((cr_event->conn_qual != SERVER_CONN_QUAL) ||\r
+ (cr_event->sp_handle.psp_handle != psp)) {\r
printf("wrong cr event data\n");\r
exit(1);\r
}\r
\r
- cr = event.event_data.cr_arrival_event_data.cr_handle;\r
+ cr = cr_event->cr_handle;\r
status = dat_cr_accept(cr, ep, 0, (DAT_PVOID)0);\r
\r
} else {\r
struct addrinfo *target;\r
- int rval;\r
\r
if (getaddrinfo (hostname, NULL, NULL, &target) != 0) {\r
printf("Error getting remote address.\n");\r
exit(1);\r
}\r
\r
- rval = ((struct sockaddr_in *)target->ai_addr)->sin_addr.s_addr;\r
printf ("Server Name: %s \n", hostname);\r
- printf ("Server Net Address: %d.%d.%d.%d\n",\r
- (rval >> 0) & 0xff,\r
- (rval >> 8) & 0xff,\r
- (rval >> 16) & 0xff,\r
- (rval >> 24) & 0xff);\r
+ printf ("Server Net Address: %s\n", inet_ntoa(\r
+ ((struct sockaddr_in *)target->ai_addr)->sin_addr));\r
\r
remote_addr = *((DAT_IA_ADDRESS_PTR)target->ai_addr);\r
-\r
- strcpy((char*)buf[ SND_RDMA_BUF_INDEX ], "client written data");\r
-\r
+ freeaddrinfo(target);\r
+ strcpy((char*)buf[ SND_RDMA_BUF_INDEX ],"client written data");\r
status = dat_ep_connect(ep,\r
&remote_addr,\r
SERVER_CONN_QUAL,\r
_OK(status, "connect dat_evd_wait");\r
\r
if (event.event_number != DAT_CONNECTION_EVENT_ESTABLISHED) {\r
- printf("unexpected event after dat_ep_connect: 0x%x\n", event.event_number); \r
+ printf("unexpected event after dat_ep_connect: 0x%x\n", \r
+ event.event_number); \r
exit(1);\r
}\r
\r
\r
/*\r
* Setup our remote memory and tell the other side about it\r
+ * Swap to network order.\r
*/\r
- printf("Sending RMR data to remote\n");\r
- r_iov.rmr_context = rmr_context[ RCV_RDMA_BUF_INDEX ];\r
- r_iov.virtual_address = (DAT_VADDR) buf[ RCV_RDMA_BUF_INDEX ];\r
- r_iov.segment_length = BUF_SIZE;\r
+ r_iov = (DAT_RMR_TRIPLET*)buf[SEND_BUF_INDEX];\r
+ r_iov->rmr_context = hton32(rmr_context[RCV_RDMA_BUF_INDEX]);\r
+ r_iov->virtual_address = hton64((DAT_VADDR)buf[RCV_RDMA_BUF_INDEX]);\r
+ r_iov->segment_length = hton32(BUF_SIZE);\r
\r
- *buf[ SEND_BUF_INDEX ] = r_iov;\r
+ printf("%d Send RMR msg to remote: r_key_ctx=0x%x,va=%p,len=0x%x\n",\r
+ getpid(), hton32(r_iov->rmr_context), \r
+ hton64(r_iov->virtual_address), hton32(r_iov->segment_length)); \r
\r
- send_msg( buf[ SEND_BUF_INDEX ],\r
- sizeof(DAT_RMR_TRIPLET),\r
- lmr_context[ SEND_BUF_INDEX ],\r
- cookie,\r
- DAT_COMPLETION_SUPPRESS_FLAG);\r
+ send_msg(buf[SEND_BUF_INDEX],\r
+ sizeof(DAT_RMR_TRIPLET),\r
+ lmr_context[SEND_BUF_INDEX],\r
+ cookie,\r
+ DAT_COMPLETION_SUPPRESS_FLAG);\r
\r
/*\r
* Wait for their RMR\r
exit(1);\r
}\r
\r
- r_iov = *buf[ RECV_BUF_INDEX ];\r
+ /* swap RMR,address info to host order */\r
+ r_iov = (DAT_RMR_TRIPLET*)buf[RECV_BUF_INDEX];\r
+ r_iov->rmr_context = ntoh32(r_iov->rmr_context);\r
+ r_iov->virtual_address = ntoh64(r_iov->virtual_address);\r
+ r_iov->segment_length = ntoh32(r_iov->segment_length);\r
\r
- printf("Received RMR from remote: r_iov: ctx=%x,va=%p,len=%d\n",\r
- r_iov.rmr_context,\r
- (void*)iov.virtual_address,\r
- r_iov.segment_length);\r
+ printf("%d Received RMR from remote: "\r
+ "r_iov: r_key_ctx=%x,va="F64x",len=0x%x\n",\r
+ getpid(), r_iov->rmr_context,\r
+ r_iov->virtual_address,\r
+ r_iov->segment_length);\r
\r
return(0);\r
}\r
\r
cookie.as_64 = 0x5555;\r
\r
- r_iov = *buf[ RECV_BUF_INDEX ];\r
+ /* RMR info already swapped back to host order in connect_ep */\r
+ r_iov = *buf[RECV_BUF_INDEX];\r
\r
- iov.lmr_context = lmr_context[ SND_RDMA_BUF_INDEX ];\r
- iov.virtual_address = (DAT_VADDR) buf[ SND_RDMA_BUF_INDEX ];\r
+ iov.lmr_context = lmr_context[SND_RDMA_BUF_INDEX];\r
+ iov.virtual_address = (DAT_VADDR) buf[SND_RDMA_BUF_INDEX];\r
iov.segment_length = BUF_SIZE;\r
\r
cookie.as_64 = 0x9999;\r
\r
status = dat_ib_post_rdma_write_immed(ep, // ep_handle\r
- 1, // num_segments\r
- &iov, // LMR\r
- cookie, // user_cookie\r
- &r_iov, // RMR\r
- immed_data,\r
- DAT_COMPLETION_DEFAULT_FLAG);\r
+ 1, // segments\r
+ &iov, // LMR\r
+ cookie, // user_cookie\r
+ &r_iov, // RMR\r
+ immed_data,\r
+ DAT_COMPLETION_DEFAULT_FLAG);\r
_OK(status, "dat_ib_post_rdma_write_immed");\r
\r
/*\r
- * Collect first event, write completion or the inbound recv with immed\r
+ * Collect first event, write completion or inbound recv with immed\r
*/\r
status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);\r
_OK(status, "dat_evd_wait after dat_ib_post_rdma_write");\r
\r
\r
/*\r
- * Collect second event, write completion or inbound recv with immed\r
+ * Collect second event, write completion or inbound recv with immed\r
*/\r
status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);\r
_OK(status, "dat_evd_wait after dat_ib_post_rdma_write");\r
\r
if ((server) && (immed_data_recv != 0x7777))\r
{\r
- printf("ERROR: Server: unexpected immed_data_recv 0x%x/0x%x\n", \r
+ printf("ERROR: Server: unexpected imm_data_recv 0x%x/0x%x\n", \r
0x7777, immed_data_recv);\r
exit(1);\r
} \r
else if ((!server) && (immed_data_recv != 0x1111))\r
{\r
- printf("ERROR: Client: unexpected immed_data_recv 0x%x/0x%x\n", \r
+ printf("ERROR: Client: unexpected imm_data_recv 0x%x/0x%x\n", \r
0x1111, immed_data_recv);\r
exit(1);\r
}\r
\r
if (server)\r
- printf("Server received immed_data=0x%x\n", immed_data_recv);\r
+ printf("Server received immed_data=0x%x\n",immed_data_recv);\r
else\r
- printf("Client received immed_data=0x%x\n", immed_data_recv);\r
+ printf("Client received immed_data=0x%x\n",immed_data_recv);\r
\r
printf("rdma buffer %p contains: %s\n", \r
buf[ RCV_RDMA_BUF_INDEX ], buf[ RCV_RDMA_BUF_INDEX ]);\r
DAT_COUNT nmore;\r
DAT_LMR_TRIPLET l_iov;\r
DAT_RMR_TRIPLET r_iov;\r
- volatile DAT_UINT64 *target = (DAT_UINT64*)buf[ RCV_RDMA_BUF_INDEX ];\r
+ volatile DAT_UINT64 *target = (DAT_UINT64*)buf[RCV_RDMA_BUF_INDEX];\r
DAT_DTO_COMPLETION_EVENT_DATA *dto_event = \r
&event.event_data.dto_completion_event_data;\r
DAT_IB_EXTENSION_EVENT_DATA *ext_event = \r
\r
printf("\nDoing CMP and SWAP\n");\r
\r
+ /* RMR info already swapped back to host order in connect_ep */\r
r_iov = *buf[ RECV_BUF_INDEX ];\r
\r
l_iov.lmr_context = lmr_atomic_context;\r
sleep(1);\r
/* server does not compare and should not swap */\r
printf("dtx svr - starting cmp_swap\n");\r
- status = dat_ib_post_cmp_and_swap( ep, \r
- (DAT_UINT64)0x654321, \r
- (DAT_UINT64)0x6789A, \r
- &l_iov,\r
- cookie, \r
- &r_iov, \r
- DAT_COMPLETION_DEFAULT_FLAG);\r
+ status = dat_ib_post_cmp_and_swap(\r
+ ep, \r
+ (DAT_UINT64)0x654321, \r
+ (DAT_UINT64)0x6789A, \r
+ &l_iov,\r
+ cookie, \r
+ &r_iov, \r
+ DAT_COMPLETION_DEFAULT_FLAG);\r
printf("dtx svr - done cmp_swap, chk status\n");\r
} else {\r
*target = 0x54321;\r
sleep(1); \r
printf("dtx cli - starting cmp_swap\n");\r
/* client does compare and should swap */\r
- status = dat_ib_post_cmp_and_swap( ep, \r
- (DAT_UINT64)0x12345, \r
- (DAT_UINT64)0x98765,\r
- &l_iov,\r
- cookie, \r
- &r_iov, \r
- DAT_COMPLETION_DEFAULT_FLAG);\r
+ status = dat_ib_post_cmp_and_swap(\r
+ ep, \r
+ (DAT_UINT64)0x12345, \r
+ (DAT_UINT64)0x98765,\r
+ &l_iov,\r
+ cookie, \r
+ &r_iov, \r
+ DAT_COMPLETION_DEFAULT_FLAG);\r
printf("dtx cli - done cmp_swap, chk status\n");\r
}\r
_OK(status, "dat_ib_post_cmp_and_swap");\r
\r
_OK(dto_event->status, "event status for CMP and SWAP");\r
if (ext_event->type != DAT_IB_CMP_AND_SWAP) {\r
- printf("unexpected event data of cmp and swap : type=%d cookie=%d "\r
- "original 0x%llx\n",\r
+ printf("unexpected event data of cmp_swap: type=%d cookie=%d "\r
+ "original "F64x"\n",\r
(int)ext_event->type,\r
(int)dto_event->user_cookie.as_64,\r
*atomic_buf);\r
sleep(2); /* wait for other side to complete swap */\r
\r
if (server) {\r
- printf("Server got original data = 0x%llx, expected "\r
+ printf("Server got original data = "F64x", expected "\r
"0x54321\n", *atomic_buf);\r
- printf("Client final result (on server) = 0x%llx, expected "\r
+ printf("Client final result (on server) = "F64x", expected "\r
"0x98765\n", *target);\r
\r
if (*atomic_buf != 0x54321 || *target != 0x98765) {\r
exit(1);\r
}\r
} else {\r
- printf("Client got original data = 0x%llx, expected "\r
+ printf("Client got original data = "F64x", expected "\r
"0x12345\n",*atomic_buf);\r
- printf("Server final result (on client) = 0x%llx, expected "\r
+ printf("Server final result (on client) = 0x"F64x", expected "\r
"0x54321\n", *target);\r
\r
if (*atomic_buf != 0x12345 || *target != 0x54321) {\r
DAT_COUNT nmore;\r
DAT_LMR_TRIPLET l_iov;\r
DAT_RMR_TRIPLET r_iov;\r
- volatile DAT_UINT64 *target = (DAT_UINT64*)buf[ RCV_RDMA_BUF_INDEX ];\r
+ volatile DAT_UINT64 *target = (DAT_UINT64*)buf[RCV_RDMA_BUF_INDEX];\r
DAT_DTO_COMPLETION_EVENT_DATA *dto_event = \r
&event.event_data.dto_completion_event_data;\r
DAT_IB_EXTENSION_EVENT_DATA *ext_event = \r
\r
printf("\nDoing FETCH and ADD\n");\r
\r
+ /* RMR info already swapped back to host order in connect_ep */\r
r_iov = *buf[ RECV_BUF_INDEX ];\r
\r
l_iov.lmr_context = lmr_atomic_context;\r
sleep(1);\r
*target = 0x10;\r
sleep(1);\r
- status = dat_ib_post_fetch_and_add( ep, \r
- (DAT_UINT64)0x100,\r
- &l_iov,\r
- cookie, \r
- &r_iov, \r
- DAT_COMPLETION_DEFAULT_FLAG);\r
+ status = dat_ib_post_fetch_and_add(\r
+ ep, \r
+ (DAT_UINT64)0x100,\r
+ &l_iov,\r
+ cookie, \r
+ &r_iov, \r
+ DAT_COMPLETION_DEFAULT_FLAG);\r
} else {\r
/* Wait for server, no swap so nothing to check */\r
*target = 0x100;\r
sleep(1);\r
- status = dat_ib_post_fetch_and_add( ep, \r
- (DAT_UINT64)0x10, \r
- &l_iov,\r
- cookie, \r
- &r_iov, \r
- DAT_COMPLETION_DEFAULT_FLAG);\r
+ status = dat_ib_post_fetch_and_add(\r
+ ep, \r
+ (DAT_UINT64)0x10, \r
+ &l_iov,\r
+ cookie, \r
+ &r_iov, \r
+ DAT_COMPLETION_DEFAULT_FLAG);\r
}\r
_OK(status, "dat_ib_post_fetch_and_add");\r
status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);\r
}\r
\r
if (server) {\r
- printf("Client original data (on server) = 0x%llx, expected "\r
+ printf("Client original data (on server) = "F64x", expected "\r
"0x100\n", *atomic_buf);\r
} else {\r
- printf("Server original data (on client) = 0x%llx, expected "\r
+ printf("Server original data (on client) = "F64x", expected "\r
"0x10\n", *atomic_buf);\r
}\r
\r
sleep(1); \r
\r
if (server) {\r
- status = dat_ib_post_fetch_and_add(ep, \r
- (DAT_UINT64)0x100, \r
- &l_iov,\r
- cookie, \r
- &r_iov, \r
- DAT_COMPLETION_DEFAULT_FLAG);\r
+ status = dat_ib_post_fetch_and_add(\r
+ ep, \r
+ (DAT_UINT64)0x100, \r
+ &l_iov,\r
+ cookie, \r
+ &r_iov, \r
+ DAT_COMPLETION_DEFAULT_FLAG);\r
} else {\r
- status = dat_ib_post_fetch_and_add(ep, \r
- (DAT_UINT64)0x10,\r
- &l_iov,\r
- cookie, \r
- &r_iov, \r
- DAT_COMPLETION_DEFAULT_FLAG);\r
+ status = dat_ib_post_fetch_and_add(\r
+ ep, \r
+ (DAT_UINT64)0x10,\r
+ &l_iov,\r
+ cookie, \r
+ &r_iov, \r
+ DAT_COMPLETION_DEFAULT_FLAG);\r
}\r
\r
status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);\r
sleep(1); /* wait for other side to complete fetch_add */\r
\r
if (server) {\r
- printf("Server got original data = 0x%llx, expected "\r
+ printf("Server got original data = "F64x", expected "\r
"0x200\n", *atomic_buf);\r
- printf("Client final result (on server) = 0x%llx, expected "\r
+ printf("Client final result (on server) = "F64x", expected "\r
"0x30\n", *target);\r
\r
if (*atomic_buf != 0x200 || *target != 0x30) {\r
exit(1);\r
}\r
} else {\r
- printf("Server side original data = 0x%llx, expected "\r
+ printf("Server side original data = "F64x", expected "\r
"0x20\n", *atomic_buf);\r
- printf("Server final result (on client) = 0x%llx, expected "\r
+ printf("Server final result (on client) = "F64x", expected "\r
"0x300\n", *target);\r
\r
if (*atomic_buf != 0x20 || *target != 0x300) {\r
\r
i = WSAStartup ( MAKEWORD(2,2), &wsaData );\r
if ( i != 0 ) {\r
- printf("%s WSAStartup(2.2) failed? (0x%x)\n",argv[0],i);\r
+ printf("%s WSAStartup(2.2) fail? (0x%x)\n",argv[0],i);\r
fflush(stdout);\r
exit(1);\r
}\r
projects / ~shefty / rdma-win.git / commitdiff