* SOFTWARE.
*
* $Id: $
- */\r
-#include <stdio.h>\r
-#include <stdlib.h>\r
-#include <netdb.h>\r
-#include <sys/socket.h>\r
-#include <netinet/in.h>\r
-#include <netinet/tcp.h>\r
-#include <arpa/inet.h>\r
-#include <endian.h>\r
-#include <byteswap.h>\r
-\r
-#include "dat/udat.h"\r
-#include "dat/dat_ib_extensions.h"\r
-\r
-#define _OK(status, str) \\r
-{ \\r
- const char *maj_msg, *min_msg; \\r
- if (status != DAT_SUCCESS) { \\r
- dat_strerror(status, &maj_msg, &min_msg); \\r
- fprintf(stderr, str " returned %s : %s\n", maj_msg, min_msg); \\r
- exit(1); \\r
- } \\r
-}\r
-\r
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <netdb.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+#include <arpa/inet.h>
+#include <endian.h>
+#include <byteswap.h>
+
+#include "dat/udat.h"
+#include "dat/dat_ib_extensions.h"
+
+#define _OK(status, str) \
+{ \
+ const char *maj_msg, *min_msg; \
+ if (status != DAT_SUCCESS) { \
+ dat_strerror(status, &maj_msg, &min_msg); \
+ fprintf(stderr, str " returned %s : %s\n", maj_msg, min_msg); \
+ exit(1); \
+ } \
+}
+
#define DTO_TIMEOUT (1000*1000*5)
#define CONN_TIMEOUT (1000*1000*10)
-#define SERVER_TIMEOUT (1000*1000*120)\r
-#define SERVER_CONN_QUAL 31111\r
-#define BUF_SIZE 256\r
-#define BUF_SIZE_ATOMIC 8\r
-#define REG_MEM_COUNT 10\r
-#define SND_RDMA_BUF_INDEX 0\r
-#define RCV_RDMA_BUF_INDEX 1\r
-#define SEND_BUF_INDEX 2\r
-#define RECV_BUF_INDEX 3\r
-\r
-u_int64_t *atomic_buf;\r
-DAT_LMR_HANDLE lmr_atomic;\r
-DAT_LMR_CONTEXT lmr_atomic_context;\r
-DAT_RMR_CONTEXT rmr_atomic_context;\r
-DAT_VLEN reg_atomic_size;\r
-DAT_VADDR reg_atomic_addr;\r
-DAT_LMR_HANDLE lmr[ REG_MEM_COUNT ];\r
-DAT_LMR_CONTEXT lmr_context[ REG_MEM_COUNT ];\r
-DAT_RMR_TRIPLET rmr[ REG_MEM_COUNT ];\r
-DAT_RMR_CONTEXT rmr_context[ REG_MEM_COUNT ];\r
-DAT_VLEN reg_size[ REG_MEM_COUNT ];\r
-DAT_VADDR reg_addr[ REG_MEM_COUNT ];\r
-DAT_RMR_TRIPLET * buf[ REG_MEM_COUNT ];\r
-DAT_EP_HANDLE ep;\r
-DAT_EVD_HANDLE async_evd = DAT_HANDLE_NULL;\r
-DAT_IA_HANDLE ia = DAT_HANDLE_NULL;\r
-DAT_PZ_HANDLE pz = DAT_HANDLE_NULL;\r
-DAT_EVD_HANDLE cr_evd = DAT_HANDLE_NULL;\r
-DAT_EVD_HANDLE con_evd = DAT_HANDLE_NULL;\r
-DAT_EVD_HANDLE dto_evd = DAT_HANDLE_NULL;\r
-DAT_PSP_HANDLE psp = DAT_HANDLE_NULL;\r
-DAT_CR_HANDLE cr = DAT_HANDLE_NULL;\r
-int server;\r
-\r
-char *usage = "-s | hostname (default == -s)\n";\r
-\r
-void\r
-send_msg(\r
- void *data,\r
- DAT_COUNT size,\r
- DAT_LMR_CONTEXT context,\r
- DAT_DTO_COOKIE cookie,\r
- DAT_COMPLETION_FLAGS flags)\r
-{\r
- DAT_LMR_TRIPLET iov;\r
- DAT_EVENT event;\r
- DAT_COUNT nmore;\r
- DAT_RETURN status;\r
- DAT_DTO_COMPLETION_EVENT_DATA *dto_event = \r
- &event.event_data.dto_completion_event_data;\r
-\r
- iov.lmr_context = context;\r
- iov.virtual_address = (DAT_VADDR)(unsigned long)data;\r
- iov.segment_length = (DAT_VLEN)size;\r
-\r
- status = dat_ep_post_send(ep,\r
- 1,\r
- &iov,\r
- cookie,\r
- flags);\r
- _OK(status, "dat_ep_post_send");\r
-\r
- if (! (flags & DAT_COMPLETION_SUPPRESS_FLAG)) {\r
- status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);\r
- _OK(status, "dat_evd_wait after dat_ep_post_send");\r
-\r
- if (event.event_number != DAT_DTO_COMPLETION_EVENT) {\r
- printf("unexpected event waiting for post_send completion - 0x%x\n", event.event_number);\r
- exit(1);\r
- }\r
-\r
- _OK(dto_event->status, "event status for post_send");\r
- }\r
-}\r
-\r
-int\r
-connect_ep(char *hostname)\r
-{\r
- DAT_SOCK_ADDR remote_addr;\r
- DAT_EP_ATTR ep_attr;\r
- DAT_RETURN status;\r
- DAT_REGION_DESCRIPTION region;\r
- DAT_EVENT event;\r
- DAT_COUNT nmore;\r
- DAT_LMR_TRIPLET iov;\r
- DAT_RMR_TRIPLET r_iov;\r
- DAT_DTO_COOKIE cookie;\r
- int i;\r
- DAT_DTO_COMPLETION_EVENT_DATA *dto_event = \r
- &event.event_data.dto_completion_event_data;\r
-\r
- status = dat_ia_open("OpenIB-cma", 8, &async_evd, &ia);\r
- _OK(status, "dat_ia_open");\r
-\r
- status = dat_pz_create(ia, &pz);\r
- _OK(status, "dat_pz_create");\r
-\r
- status = dat_evd_create(ia, 10, DAT_HANDLE_NULL, DAT_EVD_CR_FLAG, &cr_evd );\r
- _OK(status, "dat_evd_create CR");\r
- status = dat_evd_create(ia, 10, DAT_HANDLE_NULL, DAT_EVD_CONNECTION_FLAG, &con_evd );\r
- _OK(status, "dat_evd_create CR");\r
- status = dat_evd_create(ia, 10, DAT_HANDLE_NULL, DAT_EVD_DTO_FLAG, &dto_evd );\r
- _OK(status, "dat_evd_create DTO");\r
-\r
- memset(&ep_attr, 0, sizeof(ep_attr));\r
- ep_attr.service_type = DAT_SERVICE_TYPE_RC;\r
- ep_attr.max_rdma_size = 0x10000;\r
- ep_attr.qos = 0;\r
- ep_attr.recv_completion_flags = 0;\r
- ep_attr.max_recv_dtos = 10;\r
- ep_attr.max_request_dtos = 10;\r
- ep_attr.max_recv_iov = 1;\r
- ep_attr.max_request_iov = 1;\r
- ep_attr.max_rdma_read_in = 4;\r
- ep_attr.max_rdma_read_out = 4;\r
- ep_attr.request_completion_flags = DAT_COMPLETION_DEFAULT_FLAG;\r
- ep_attr.ep_transport_specific_count = 0;\r
- ep_attr.ep_transport_specific = NULL;\r
- ep_attr.ep_provider_specific_count = 0;\r
- ep_attr.ep_provider_specific = NULL;\r
-\r
- status = dat_ep_create(ia, pz, dto_evd, dto_evd, con_evd, &ep_attr, &ep);\r
- _OK(status, "dat_ep_create");\r
-\r
- for (i = 0; i < REG_MEM_COUNT; i++) {\r
- buf[ i ] = (DAT_RMR_TRIPLET*)malloc(BUF_SIZE);\r
- region.for_va = buf[ i ];\r
- status = dat_lmr_create(ia,\r
- DAT_MEM_TYPE_VIRTUAL,\r
- region,\r
- BUF_SIZE,\r
- pz,\r
- DAT_MEM_PRIV_ALL_FLAG|DAT_IB_MEM_PRIV_REMOTE_ATOMIC,\r
+#define SERVER_TIMEOUT (1000*1000*120)
+#define SERVER_CONN_QUAL 31111
+#define BUF_SIZE 256
+#define BUF_SIZE_ATOMIC 8
+#define REG_MEM_COUNT 10
+#define SND_RDMA_BUF_INDEX 0
+#define RCV_RDMA_BUF_INDEX 1
+#define SEND_BUF_INDEX 2
+#define RECV_BUF_INDEX 3
+
+u_int64_t *atomic_buf;
+DAT_LMR_HANDLE lmr_atomic;
+DAT_LMR_CONTEXT lmr_atomic_context;
+DAT_RMR_CONTEXT rmr_atomic_context;
+DAT_VLEN reg_atomic_size;
+DAT_VADDR reg_atomic_addr;
+DAT_LMR_HANDLE lmr[ REG_MEM_COUNT ];
+DAT_LMR_CONTEXT lmr_context[ REG_MEM_COUNT ];
+DAT_RMR_TRIPLET rmr[ REG_MEM_COUNT ];
+DAT_RMR_CONTEXT rmr_context[ REG_MEM_COUNT ];
+DAT_VLEN reg_size[ REG_MEM_COUNT ];
+DAT_VADDR reg_addr[ REG_MEM_COUNT ];
+DAT_RMR_TRIPLET * buf[ REG_MEM_COUNT ];
+DAT_EP_HANDLE ep;
+DAT_EVD_HANDLE async_evd = DAT_HANDLE_NULL;
+DAT_IA_HANDLE ia = DAT_HANDLE_NULL;
+DAT_PZ_HANDLE pz = DAT_HANDLE_NULL;
+DAT_EVD_HANDLE cr_evd = DAT_HANDLE_NULL;
+DAT_EVD_HANDLE con_evd = DAT_HANDLE_NULL;
+DAT_EVD_HANDLE dto_evd = DAT_HANDLE_NULL;
+DAT_PSP_HANDLE psp = DAT_HANDLE_NULL;
+DAT_CR_HANDLE cr = DAT_HANDLE_NULL;
+int server;
+
+char *usage = "-s | hostname (default == -s)\n";
+
+void
+send_msg(
+ void *data,
+ DAT_COUNT size,
+ DAT_LMR_CONTEXT context,
+ DAT_DTO_COOKIE cookie,
+ DAT_COMPLETION_FLAGS flags)
+{
+ DAT_LMR_TRIPLET iov;
+ DAT_EVENT event;
+ DAT_COUNT nmore;
+ DAT_RETURN status;
+ DAT_DTO_COMPLETION_EVENT_DATA *dto_event =
+ &event.event_data.dto_completion_event_data;
+
+ iov.lmr_context = context;
+ iov.virtual_address = (DAT_VADDR)(unsigned long)data;
+ iov.segment_length = (DAT_VLEN)size;
+
+ status = dat_ep_post_send(ep,
+ 1,
+ &iov,
+ cookie,
+ flags);
+ _OK(status, "dat_ep_post_send");
+
+ if (! (flags & DAT_COMPLETION_SUPPRESS_FLAG)) {
+ status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
+ _OK(status, "dat_evd_wait after dat_ep_post_send");
+
+ if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
+ printf("unexpected event waiting for post_send completion - 0x%x\n", event.event_number);
+ exit(1);
+ }
+
+ _OK(dto_event->status, "event status for post_send");
+ }
+}
+
+int
+connect_ep(char *hostname)
+{
+ DAT_SOCK_ADDR remote_addr;
+ DAT_EP_ATTR ep_attr;
+ DAT_RETURN status;
+ DAT_REGION_DESCRIPTION region;
+ DAT_EVENT event;
+ DAT_COUNT nmore;
+ DAT_LMR_TRIPLET iov;
+ DAT_RMR_TRIPLET r_iov;
+ DAT_DTO_COOKIE cookie;
+ int i;
+ DAT_DTO_COMPLETION_EVENT_DATA *dto_event =
+ &event.event_data.dto_completion_event_data;
+
+ status = dat_ia_open("OpenIB-2-cma", 8, &async_evd, &ia);
+ _OK(status, "dat_ia_open");
+
+ status = dat_pz_create(ia, &pz);
+ _OK(status, "dat_pz_create");
+
+ status = dat_evd_create(ia, 10, DAT_HANDLE_NULL, DAT_EVD_CR_FLAG, &cr_evd );
+ _OK(status, "dat_evd_create CR");
+ status = dat_evd_create(ia, 10, DAT_HANDLE_NULL, DAT_EVD_CONNECTION_FLAG, &con_evd );
+ _OK(status, "dat_evd_create CR");
+ status = dat_evd_create(ia, 10, DAT_HANDLE_NULL, DAT_EVD_DTO_FLAG, &dto_evd );
+ _OK(status, "dat_evd_create DTO");
+
+ memset(&ep_attr, 0, sizeof(ep_attr));
+ ep_attr.service_type = DAT_SERVICE_TYPE_RC;
+ ep_attr.max_rdma_size = 0x10000;
+ ep_attr.qos = 0;
+ ep_attr.recv_completion_flags = 0;
+ ep_attr.max_recv_dtos = 10;
+ ep_attr.max_request_dtos = 10;
+ ep_attr.max_recv_iov = 1;
+ ep_attr.max_request_iov = 1;
+ ep_attr.max_rdma_read_in = 4;
+ ep_attr.max_rdma_read_out = 4;
+ ep_attr.request_completion_flags = DAT_COMPLETION_DEFAULT_FLAG;
+ ep_attr.ep_transport_specific_count = 0;
+ ep_attr.ep_transport_specific = NULL;
+ ep_attr.ep_provider_specific_count = 0;
+ ep_attr.ep_provider_specific = NULL;
+
+ status = dat_ep_create(ia, pz, dto_evd, dto_evd, con_evd, &ep_attr, &ep);
+ _OK(status, "dat_ep_create");
+
+ for (i = 0; i < REG_MEM_COUNT; i++) {
+ buf[ i ] = (DAT_RMR_TRIPLET*)malloc(BUF_SIZE);
+ region.for_va = buf[ i ];
+ status = dat_lmr_create(ia,
+ DAT_MEM_TYPE_VIRTUAL,
+ region,
+ BUF_SIZE,
+ pz,
+ DAT_MEM_PRIV_ALL_FLAG|DAT_IB_MEM_PRIV_REMOTE_ATOMIC,
DAT_VA_TYPE_VA,
- &lmr[ i ],\r
- &lmr_context[ i ],\r
- &rmr_context[ i ],\r
- ®_size[ i ],\r
- ®_addr[ i ]);\r
- _OK(status, "dat_lmr_create");\r
- }\r
-\r
- /* register atomic return buffer for original data */\r
- atomic_buf = (u_int64_t*)malloc(BUF_SIZE);\r
- region.for_va = atomic_buf;\r
- status = dat_lmr_create(ia,\r
- DAT_MEM_TYPE_VIRTUAL,\r
- region,\r
- BUF_SIZE_ATOMIC,\r
- pz,\r
- DAT_MEM_PRIV_ALL_FLAG|DAT_IB_MEM_PRIV_REMOTE_ATOMIC,\r
+ &lmr[ i ],
+ &lmr_context[ i ],
+ &rmr_context[ i ],
+ ®_size[ i ],
+ ®_addr[ i ]);
+ _OK(status, "dat_lmr_create");
+ }
+
+ /* register atomic return buffer for original data */
+ atomic_buf = (u_int64_t*)malloc(BUF_SIZE);
+ region.for_va = atomic_buf;
+ status = dat_lmr_create(ia,
+ DAT_MEM_TYPE_VIRTUAL,
+ region,
+ BUF_SIZE_ATOMIC,
+ pz,
+ DAT_MEM_PRIV_ALL_FLAG|DAT_IB_MEM_PRIV_REMOTE_ATOMIC,
DAT_VA_TYPE_VA,
- &lmr_atomic,\r
- &lmr_atomic_context,\r
- &rmr_atomic_context,\r
- ®_atomic_size,\r
- ®_atomic_addr);\r
- _OK(status, "dat_lmr_create atomic");\r
- \r
- for (i = RECV_BUF_INDEX; i < REG_MEM_COUNT; i++) {\r
- cookie.as_64 = i;\r
- iov.lmr_context = lmr_context[ i ];\r
- iov.virtual_address = (DAT_VADDR)(unsigned long)buf[ i ];\r
- iov.segment_length = BUF_SIZE;\r
-\r
- status = dat_ep_post_recv(ep,\r
- 1,\r
- &iov,\r
- cookie,\r
- DAT_COMPLETION_DEFAULT_FLAG);\r
- _OK(status, "dat_ep_post_recv");\r
- }\r
-\r
- /* setup receive buffer to initial string to be overwritten */\r
- strcpy((char*)buf[ RCV_RDMA_BUF_INDEX ], "blah, blah, blah\n");\r
-\r
- if (server) {\r
-\r
- strcpy((char*)buf[ SND_RDMA_BUF_INDEX ], "server written data");\r
-\r
- status = dat_psp_create(ia,\r
- SERVER_CONN_QUAL,\r
- cr_evd,\r
- DAT_PSP_CONSUMER_FLAG,\r
- &psp);\r
- _OK(status, "dat_psp_create");\r
-\r
- printf("Server waiting for connect request\n");\r
- status = dat_evd_wait(cr_evd, SERVER_TIMEOUT, 1, &event, &nmore);\r
- _OK(status, "listen dat_evd_wait");\r
-\r
- if (event.event_number != DAT_CONNECTION_REQUEST_EVENT) {\r
- printf("unexpected event after dat_psp_create: 0x%x\n", event.event_number); \r
- exit(1);\r
- }\r
-\r
- if ((event.event_data.cr_arrival_event_data.conn_qual != SERVER_CONN_QUAL) ||\r
- (event.event_data.cr_arrival_event_data.sp_handle.psp_handle != psp)) {\r
-\r
- printf("wrong cr event data\n");\r
- exit(1);\r
- }\r
-\r
- cr = event.event_data.cr_arrival_event_data.cr_handle;\r
- status = dat_cr_accept(cr, ep, 0, (DAT_PVOID)0);\r
-\r
- } else {\r
- struct addrinfo *target;\r
- int rval;\r
-\r
- if (getaddrinfo (hostname, NULL, NULL, &target) != 0) {\r
- printf("Error getting remote address.\n");\r
- exit(1);\r
- }\r
-\r
- rval = ((struct sockaddr_in *)target->ai_addr)->sin_addr.s_addr;\r
- printf ("Server Name: %s \n", hostname);\r
- printf ("Server Net Address: %d.%d.%d.%d\n",\r
- (rval >> 0) & 0xff,\r
- (rval >> 8) & 0xff,\r
- (rval >> 16) & 0xff,\r
- (rval >> 24) & 0xff);\r
-\r
- remote_addr = *((DAT_IA_ADDRESS_PTR)target->ai_addr);\r
-\r
- strcpy((char*)buf[ SND_RDMA_BUF_INDEX ], "client written data");\r
-\r
- status = dat_ep_connect(ep,\r
- &remote_addr,\r
- SERVER_CONN_QUAL,\r
- CONN_TIMEOUT,\r
- 0,\r
- (DAT_PVOID)0,\r
- 0,\r
- DAT_CONNECT_DEFAULT_FLAG );\r
- _OK(status, "dat_psp_create");\r
- }\r
-\r
- printf("Client waiting for connect response\n");\r
- status = dat_evd_wait(con_evd, CONN_TIMEOUT, 1, &event, &nmore);\r
- _OK(status, "connect dat_evd_wait");\r
-\r
- if (event.event_number != DAT_CONNECTION_EVENT_ESTABLISHED) {\r
- printf("unexpected event after dat_ep_connect: 0x%x\n", event.event_number); \r
- exit(1);\r
- }\r
-\r
- printf("Connected!\n");\r
-\r
- /*\r
- * Setup our remote memory and tell the other side about it\r
- */\r
- printf("Sending RMR data to remote\n");\r
- r_iov.rmr_context = rmr_context[ RCV_RDMA_BUF_INDEX ];\r
- r_iov.virtual_address = (DAT_VADDR)((unsigned long)buf[ RCV_RDMA_BUF_INDEX ]);\r
- r_iov.segment_length = BUF_SIZE;\r
-\r
- *buf[ SEND_BUF_INDEX ] = r_iov;\r
-\r
- send_msg( buf[ SEND_BUF_INDEX ],\r
- sizeof(DAT_RMR_TRIPLET),\r
- lmr_context[ SEND_BUF_INDEX ],\r
- cookie,\r
- DAT_COMPLETION_SUPPRESS_FLAG);\r
-\r
- /*\r
- * Wait for their RMR\r
- */\r
- printf("Waiting for remote to send RMR data\n");\r
- status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);\r
- _OK(status, "dat_evd_wait after dat_ep_post_send");\r
-\r
- if (event.event_number != DAT_DTO_COMPLETION_EVENT) {\r
- printf("unexpected event waiting for RMR context - 0x%x\n", \r
- event.event_number);\r
- exit(1);\r
- }\r
-\r
- _OK(dto_event->status, "event status for post_send");\r
- if ((dto_event->transfered_length != sizeof(DAT_RMR_TRIPLET)) ||\r
- (dto_event->user_cookie.as_64 != RECV_BUF_INDEX)) {\r
- printf("unexpected event data for receive: len=%d cookie=%d expected %d/%d\n",\r
- (int)dto_event->transfered_length,\r
- (int)dto_event->user_cookie.as_64,\r
- sizeof(DAT_RMR_TRIPLET), RECV_BUF_INDEX);\r
- exit(1);\r
- }\r
-\r
- r_iov = *buf[ RECV_BUF_INDEX ];\r
-\r
- printf("Received RMR from remote: r_iov: ctx=%x,va=%p,len=%d\n",\r
- r_iov.rmr_context,\r
- (void*)(unsigned long)r_iov.virtual_address,\r
- r_iov.segment_length);\r
-\r
- return(0);\r
-}\r
-\r
-int\r
-disconnect_ep()\r
-{\r
- DAT_RETURN status;\r
- int i;\r
- DAT_EVENT event;\r
- DAT_COUNT nmore;\r
-\r
- status = dat_ep_disconnect(ep, DAT_CLOSE_DEFAULT);\r
- _OK(status, "dat_ep_disconnect");\r
-\r
+ &lmr_atomic,
+ &lmr_atomic_context,
+ &rmr_atomic_context,
+ ®_atomic_size,
+ ®_atomic_addr);
+ _OK(status, "dat_lmr_create atomic");
+
+ for (i = RECV_BUF_INDEX; i < REG_MEM_COUNT; i++) {
+ cookie.as_64 = i;
+ iov.lmr_context = lmr_context[ i ];
+ iov.virtual_address = (DAT_VADDR)(unsigned long)buf[ i ];
+ iov.segment_length = BUF_SIZE;
+
+ status = dat_ep_post_recv(ep,
+ 1,
+ &iov,
+ cookie,
+ DAT_COMPLETION_DEFAULT_FLAG);
+ _OK(status, "dat_ep_post_recv");
+ }
+
+ /* setup receive buffer to initial string to be overwritten */
+ strcpy((char*)buf[ RCV_RDMA_BUF_INDEX ], "blah, blah, blah\n");
+
+ if (server) {
+
+ strcpy((char*)buf[ SND_RDMA_BUF_INDEX ], "server written data");
+
+ status = dat_psp_create(ia,
+ SERVER_CONN_QUAL,
+ cr_evd,
+ DAT_PSP_CONSUMER_FLAG,
+ &psp);
+ _OK(status, "dat_psp_create");
+
+ printf("Server waiting for connect request\n");
+ status = dat_evd_wait(cr_evd, SERVER_TIMEOUT, 1, &event, &nmore);
+ _OK(status, "listen dat_evd_wait");
+
+ if (event.event_number != DAT_CONNECTION_REQUEST_EVENT) {
+ printf("unexpected event after dat_psp_create: 0x%x\n", event.event_number);
+ exit(1);
+ }
+
+ if ((event.event_data.cr_arrival_event_data.conn_qual != SERVER_CONN_QUAL) ||
+ (event.event_data.cr_arrival_event_data.sp_handle.psp_handle != psp)) {
+
+ printf("wrong cr event data\n");
+ exit(1);
+ }
+
+ cr = event.event_data.cr_arrival_event_data.cr_handle;
+ status = dat_cr_accept(cr, ep, 0, (DAT_PVOID)0);
+
+ } else {
+ struct addrinfo *target;
+ int rval;
+
+ if (getaddrinfo (hostname, NULL, NULL, &target) != 0) {
+ printf("Error getting remote address.\n");
+ exit(1);
+ }
+
+ rval = ((struct sockaddr_in *)target->ai_addr)->sin_addr.s_addr;
+ printf ("Server Name: %s \n", hostname);
+ printf ("Server Net Address: %d.%d.%d.%d\n",
+ (rval >> 0) & 0xff,
+ (rval >> 8) & 0xff,
+ (rval >> 16) & 0xff,
+ (rval >> 24) & 0xff);
+
+ remote_addr = *((DAT_IA_ADDRESS_PTR)target->ai_addr);
+
+ strcpy((char*)buf[ SND_RDMA_BUF_INDEX ], "client written data");
+
+ status = dat_ep_connect(ep,
+ &remote_addr,
+ SERVER_CONN_QUAL,
+ CONN_TIMEOUT,
+ 0,
+ (DAT_PVOID)0,
+ 0,
+ DAT_CONNECT_DEFAULT_FLAG );
+ _OK(status, "dat_psp_create");
+ }
+
+ printf("Client waiting for connect response\n");
+ status = dat_evd_wait(con_evd, CONN_TIMEOUT, 1, &event, &nmore);
+ _OK(status, "connect dat_evd_wait");
+
+ if (event.event_number != DAT_CONNECTION_EVENT_ESTABLISHED) {
+ printf("unexpected event after dat_ep_connect: 0x%x\n", event.event_number);
+ exit(1);
+ }
+
+ printf("Connected!\n");
+
+ /*
+ * Setup our remote memory and tell the other side about it
+ */
+ printf("Sending RMR data to remote\n");
+ r_iov.rmr_context = rmr_context[ RCV_RDMA_BUF_INDEX ];
+ r_iov.virtual_address = (DAT_VADDR)((unsigned long)buf[ RCV_RDMA_BUF_INDEX ]);
+ r_iov.segment_length = BUF_SIZE;
+
+ *buf[ SEND_BUF_INDEX ] = r_iov;
+
+ send_msg( buf[ SEND_BUF_INDEX ],
+ sizeof(DAT_RMR_TRIPLET),
+ lmr_context[ SEND_BUF_INDEX ],
+ cookie,
+ DAT_COMPLETION_SUPPRESS_FLAG);
+
+ /*
+ * Wait for their RMR
+ */
+ printf("Waiting for remote to send RMR data\n");
+ status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
+ _OK(status, "dat_evd_wait after dat_ep_post_send");
+
+ if (event.event_number != DAT_DTO_COMPLETION_EVENT) {
+ printf("unexpected event waiting for RMR context - 0x%x\n",
+ event.event_number);
+ exit(1);
+ }
+
+ _OK(dto_event->status, "event status for post_send");
+ if ((dto_event->transfered_length != sizeof(DAT_RMR_TRIPLET)) ||
+ (dto_event->user_cookie.as_64 != RECV_BUF_INDEX)) {
+ printf("unexpected event data for receive: len=%d cookie=%d expected %d/%d\n",
+ (int)dto_event->transfered_length,
+ (int)dto_event->user_cookie.as_64,
+ sizeof(DAT_RMR_TRIPLET), RECV_BUF_INDEX);
+ exit(1);
+ }
+
+ r_iov = *buf[ RECV_BUF_INDEX ];
+
+ printf("Received RMR from remote: r_iov: ctx=%x,va=%p,len=%d\n",
+ r_iov.rmr_context,
+ (void*)(unsigned long)r_iov.virtual_address,
+ r_iov.segment_length);
+
+ return(0);
+}
+
+int
+disconnect_ep()
+{
+ DAT_RETURN status;
+ int i;
+ DAT_EVENT event;
+ DAT_COUNT nmore;
+
+ status = dat_ep_disconnect(ep, DAT_CLOSE_DEFAULT);
+ _OK(status, "dat_ep_disconnect");
+
status = dat_evd_wait(con_evd, DAT_TIMEOUT_INFINITE, 1, &event, &nmore);
_OK(status, "dat_ep_disconnect");
- \r
- if (server) {\r
- status = dat_psp_free(psp);\r
- _OK(status, "dat_psp_free");\r
- }\r
-\r
- for (i = 0; i < REG_MEM_COUNT; i++) {\r
- status = dat_lmr_free(lmr[ i ]);\r
- _OK(status, "dat_lmr_free");\r
- }\r
-\r
- status = dat_lmr_free(lmr_atomic);\r
- _OK(status, "dat_lmr_free_atomic");\r
-\r
- status = dat_ep_free(ep);\r
- _OK(status, "dat_ep_free");\r
-\r
- status = dat_evd_free(dto_evd);\r
- _OK(status, "dat_evd_free DTO");\r
- status = dat_evd_free(con_evd);\r
- _OK(status, "dat_evd_free CON");\r
- status = dat_evd_free(cr_evd);\r
- _OK(status, "dat_evd_free CR");\r
-\r
- status = dat_pz_free(pz);\r
- _OK(status, "dat_pz_free");\r
-\r
- status = dat_ia_close(ia, DAT_CLOSE_DEFAULT);\r
- _OK(status, "dat_ia_close");\r
-\r
- return(0);\r
-}\r
-\r
-int\r
-do_immediate()\r
-{\r
- DAT_REGION_DESCRIPTION region;\r
- DAT_EVENT event;\r
- DAT_COUNT nmore;\r
- DAT_LMR_TRIPLET iov;\r
- DAT_RMR_TRIPLET r_iov;\r
- DAT_DTO_COOKIE cookie;\r
- DAT_RMR_CONTEXT their_context;\r
- DAT_RETURN status;\r
- DAT_UINT32 immed_data;\r
- DAT_UINT32 immed_data_recv;\r
- DAT_DTO_COMPLETION_EVENT_DATA *dto_event = \r
- &event.event_data.dto_completion_event_data;\r
- DAT_IB_EXTENSION_EVENT_DATA *ext_event = \r
- (DAT_IB_EXTENSION_EVENT_DATA *)&event.event_extension_data[0];\r
-\r
- printf("\nDoing RDMA WRITE IMMEDIATE DATA\n");\r
-\r
- if (server) {\r
- immed_data = 0x1111;\r
- } else {\r
- immed_data = 0x7777;\r
- }\r
-\r
- cookie.as_64 = 0x5555;\r
- \r
- r_iov = *buf[ RECV_BUF_INDEX ];\r
-\r
- iov.lmr_context = lmr_context[ SND_RDMA_BUF_INDEX ];\r
- iov.virtual_address = (DAT_VADDR)(unsigned long)buf[ SND_RDMA_BUF_INDEX ];\r
- iov.segment_length = BUF_SIZE;\r
-\r
- cookie.as_64 = 0x9999;\r
- \r
- status = dat_ib_post_rdma_write_immed(ep, // ep_handle\r
- 1, // num_segments\r
- &iov, // LMR\r
- cookie, // user_cookie\r
- &r_iov, // RMR\r
- immed_data,\r
- DAT_COMPLETION_DEFAULT_FLAG);\r
- _OK(status, "dat_ib_post_rdma_write_immed");\r
-\r
- /*\r
- * Collect first event, write completion or the inbound recv with immed\r
- */\r
- status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);\r
- _OK(status, "dat_evd_wait after dat_ib_post_rdma_write");\r
- if (event.event_number != DAT_IB_DTO_EVENT)\r
- {\r
- printf("unexpected event # waiting for WR-IMMED - 0x%x\n", \r
- event.event_number);\r
- exit(1);\r
- }\r
- \r
- _OK(dto_event->status, "event status");\r
- if (ext_event->type == DAT_IB_RDMA_WRITE_IMMED)\r
- {\r
- if ((dto_event->transfered_length != BUF_SIZE) ||\r
- (dto_event->user_cookie.as_64 != 0x9999)) \r
- {\r
- printf("unexpected event data for rdma_write_immed: len=%d cookie=0x%x\n",\r
- (int)dto_event->transfered_length,\r
- (int)dto_event->user_cookie.as_64);\r
- exit(1);\r
- }\r
- } \r
- else if (ext_event->type == DAT_IB_RDMA_WRITE_IMMED_DATA)\r
- {\r
- if ((dto_event->transfered_length != BUF_SIZE) ||\r
- (dto_event->user_cookie.as_64 != RECV_BUF_INDEX+1)) \r
- {\r
- printf("unexpected event data of immediate write: len=%d cookie=%d expected %d/%d\n",\r
- (int)dto_event->transfered_length,\r
- (int)dto_event->user_cookie.as_64,\r
- sizeof(int), RECV_BUF_INDEX+1);\r
- exit(1);\r
- }\r
- \r
- /* get immediate data from event */\r
- immed_data_recv = ext_event->val.immed.data;\r
- }\r
- else\r
- {\r
- printf("unexpected extension type for event - 0x%x, 0x%x\n", \r
- event.event_number, ext_event->type);\r
- exit(1);\r
- }\r
-\r
-\r
- /*\r
- * Collect second event, write completion or the inbound recv with immed\r
- */\r
- status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);\r
- _OK(status, "dat_evd_wait after dat_ib_post_rdma_write");\r
- if (event.event_number != DAT_IB_DTO_EVENT)\r
- {\r
- printf("unexpected event # waiting for WR-IMMED - 0x%x\n", \r
- event.event_number);\r
- exit(1);\r
- }\r
- \r
- _OK(dto_event->status, "event status");\r
- if (ext_event->type == DAT_IB_RDMA_WRITE_IMMED)\r
- {\r
- if ((dto_event->transfered_length != BUF_SIZE) ||\r
- (dto_event->user_cookie.as_64 != 0x9999)) \r
- {\r
- printf("unexpected event data for rdma_write_immed: len=%d cookie=0x%x\n",\r
- (int)dto_event->transfered_length,\r
- (int)dto_event->user_cookie.as_64);\r
- exit(1);\r
- }\r
- } \r
- else if (ext_event->type == DAT_IB_RDMA_WRITE_IMMED_DATA)\r
- {\r
- if ((dto_event->transfered_length != BUF_SIZE) ||\r
- (dto_event->user_cookie.as_64 != RECV_BUF_INDEX+1)) \r
- {\r
- printf("unexpected event data of immediate write: len=%d cookie=%d expected %d/%d\n",\r
- (int)dto_event->transfered_length,\r
- (int)dto_event->user_cookie.as_64,\r
- sizeof(int), RECV_BUF_INDEX+1);\r
- exit(1);\r
- }\r
- \r
- /* get immediate data from event */\r
- immed_data_recv = ext_event->val.immed.data;\r
- }\r
- else\r
- {\r
- printf("unexpected extension type for event - 0x%x, 0x%x\n", \r
- event.event_number, ext_event->type);\r
- exit(1);\r
- }\r
- \r
- if ((server) && (immed_data_recv != 0x7777))\r
- {\r
- printf("ERROR: Server got unexpected immed_data_recv 0x%x/0x%x\n", \r
- 0x7777, immed_data_recv);\r
- exit(1);\r
- } \r
- else if ((!server) && (immed_data_recv != 0x1111))\r
- {\r
- printf("ERROR: Client got unexpected immed_data_recv 0x%x/0x%x\n", \r
- 0x1111, immed_data_recv);\r
- exit(1);\r
- }\r
-\r
- if (server)\r
- printf("Server received immed_data=0x%x\n", immed_data_recv);\r
- else\r
- printf("Client received immed_data=0x%x\n", immed_data_recv);\r
- \r
- printf("rdma buffer %p contains: %s\n", \r
- buf[ RCV_RDMA_BUF_INDEX ], buf[ RCV_RDMA_BUF_INDEX ]);\r
-\r
- printf("\n RDMA_WRITE_WITH_IMMEDIATE_DATA test - PASSED\n");\r
- return (0);\r
-}\r
-\r
-int\r
-do_cmp_swap()\r
-{\r
- DAT_DTO_COOKIE cookie;\r
- DAT_RETURN status;\r
- DAT_EVENT event;\r
- DAT_COUNT nmore;\r
- DAT_LMR_TRIPLET l_iov;\r
- DAT_RMR_TRIPLET r_iov;\r
- volatile DAT_UINT64 *target = (DAT_UINT64*)buf[ RCV_RDMA_BUF_INDEX ];\r
- DAT_DTO_COMPLETION_EVENT_DATA *dto_event = \r
- &event.event_data.dto_completion_event_data;\r
- DAT_IB_EXTENSION_EVENT_DATA *ext_event = \r
- (DAT_IB_EXTENSION_EVENT_DATA *)&event.event_extension_data[0];\r
-\r
- printf("\nDoing CMP and SWAP\n");\r
- \r
- r_iov = *buf[ RECV_BUF_INDEX ];\r
- \r
- l_iov.lmr_context = lmr_atomic_context;\r
- l_iov.virtual_address = (DAT_VADDR)(unsigned long)atomic_buf;\r
- l_iov.segment_length = BUF_SIZE_ATOMIC;\r
-\r
- cookie.as_64 = 3333;\r
- if (server) {\r
- *target = 0x12345;\r
- sleep(1);\r
- /* server does not compare and should not swap */\r
- status = dat_ib_post_cmp_and_swap( ep, \r
- (DAT_UINT64)0x654321, \r
- (DAT_UINT64)0x6789A, \r
- &l_iov,\r
- cookie, \r
- &r_iov, \r
- DAT_COMPLETION_DEFAULT_FLAG);\r
- } else {\r
- *target = 0x54321;\r
- sleep(1); \r
- /* client does compare and should swap */\r
- status = dat_ib_post_cmp_and_swap( ep, \r
- (DAT_UINT64)0x12345, \r
- (DAT_UINT64)0x98765,\r
- &l_iov,\r
- cookie, \r
- &r_iov, \r
- DAT_COMPLETION_DEFAULT_FLAG);\r
- }\r
- _OK(status, "dat_ib_post_cmp_and_swap");\r
- status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);\r
- _OK(status, "dat_evd_wait for compare and swap");\r
- if (event.event_number != DAT_IB_DTO_EVENT) {\r
- printf("unexpected event after post_cmp_and_swap: 0x%x\n", \r
- event.event_number); \r
- exit(1);\r
- }\r
-\r
- _OK(dto_event->status, "event status for CMP and SWAP");\r
- if (ext_event->type != DAT_IB_CMP_AND_SWAP) {\r
- printf("unexpected event data of cmp and swap : type=%d cookie=%d original 0x%llx\n",\r
- (int)ext_event->type,\r
- (int)dto_event->user_cookie.as_64,\r
- *atomic_buf);\r
- exit(1);\r
- }\r
- sleep(1); /* wait for other side to complete swap */\r
- if (server) {\r
- printf("Server got original data = 0x%llx, expected 0x54321\n", *atomic_buf);\r
- printf("Client final result (on server) = 0x%llx, expected 0x98765\n", *target);\r
-\r
- if (*atomic_buf != 0x54321 || *target != 0x98765) {\r
- printf("ERROR: Server CMP_SWAP\n");\r
- exit(1);\r
- }\r
- } else {\r
- printf("Client got original data = 0x%llx, expected 0x12345\n",*atomic_buf);\r
- printf("Server final result (on client) = 0x%llx, expected 0x54321\n", *target);\r
-\r
- if (*atomic_buf != 0x12345 || *target != 0x54321) {\r
- printf("ERROR: Client CMP_SWAP\n");\r
- exit(1);\r
- }\r
- }\r
- printf("\n CMP_SWAP test - PASSED\n");\r
- return(0);\r
-}\r
-\r
-int\r
-do_fetch_add()\r
-{\r
- DAT_DTO_COOKIE cookie;\r
- DAT_RETURN status;\r
- DAT_EVENT event;\r
- DAT_COUNT nmore;\r
- DAT_LMR_TRIPLET l_iov;\r
- DAT_RMR_TRIPLET r_iov;\r
- volatile DAT_UINT64 *target = (DAT_UINT64*)buf[ RCV_RDMA_BUF_INDEX ];\r
- DAT_DTO_COMPLETION_EVENT_DATA *dto_event = \r
- &event.event_data.dto_completion_event_data;\r
- DAT_IB_EXTENSION_EVENT_DATA *ext_event = \r
- (DAT_IB_EXTENSION_EVENT_DATA *)&event.event_extension_data[0];\r
-\r
- printf("\nDoing FETCH and ADD\n");\r
-\r
- r_iov = *buf[ RECV_BUF_INDEX ];\r
- \r
- l_iov.lmr_context = lmr_atomic_context;\r
- l_iov.virtual_address = (DAT_VADDR)(unsigned long)atomic_buf;\r
- l_iov.segment_length = BUF_SIZE_ATOMIC;\r
-\r
- cookie.as_64 = 0x7777;\r
- if (server) {\r
- /* Wait for client to finish cmp_swap */\r
- while (*target != 0x98765)\r
- sleep(1);\r
- *target = 0x10;\r
- sleep(1);\r
- status = dat_ib_post_fetch_and_add( ep, \r
- (DAT_UINT64)0x100,\r
- &l_iov,\r
- cookie, \r
- &r_iov, \r
- DAT_COMPLETION_DEFAULT_FLAG);\r
- } else {\r
- /* Wait for server, no swap so nothing to check */\r
- *target = 0x100;\r
- sleep(1);\r
- status = dat_ib_post_fetch_and_add( ep, \r
- (DAT_UINT64)0x10, \r
- &l_iov,\r
- cookie, \r
- &r_iov, \r
- DAT_COMPLETION_DEFAULT_FLAG);\r
- }\r
- _OK(status, "dat_ib_post_fetch_and_add");\r
- status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);\r
- _OK(status, "dat_evd_wait for fetch and add");\r
- if (event.event_number != DAT_IB_DTO_EVENT) {\r
- printf("unexpected event after post_fetch_and_add: 0x%x\n", event.event_number); \r
- exit(1);\r
- }\r
-\r
- _OK(dto_event->status, "event status for FETCH and ADD");\r
- if (ext_event->type != DAT_IB_FETCH_AND_ADD) {\r
- printf("unexpected event data of fetch and add : type=%d cookie=%d original%d\n",\r
- (int)ext_event->type,\r
- (int)dto_event->user_cookie.as_64,\r
- (int)*atomic_buf);\r
- exit(1);\r
- }\r
-\r
- if (server) {\r
- printf("Client original data (on server) = 0x%llx, expected 0x100\n", *atomic_buf);\r
- } else {\r
- printf("Server original data (on client) = 0x%llx, expected 0x10\n", *atomic_buf);\r
- }\r
-\r
- sleep(1); \r
-\r
- if (server) {\r
- status = dat_ib_post_fetch_and_add( ep, \r
- (DAT_UINT64)0x100, \r
- &l_iov,\r
- cookie, \r
- &r_iov, \r
- DAT_COMPLETION_DEFAULT_FLAG);\r
- } else {\r
- status = dat_ib_post_fetch_and_add( ep, \r
- (DAT_UINT64)0x10,\r
- &l_iov,\r
- cookie, \r
- &r_iov, \r
- DAT_COMPLETION_DEFAULT_FLAG);\r
- }\r
-\r
- status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);\r
- _OK(status, "dat_evd_wait for second fetch and add");\r
- if (event.event_number != DAT_IB_DTO_EVENT) {\r
- printf("unexpected event after second post_fetch_and_add: 0x%x\n", event.event_number); \r
- exit(1);\r
- }\r
-\r
- _OK(dto_event->status, "event status for second FETCH and ADD");\r
- if (ext_event->type != DAT_IB_FETCH_AND_ADD) {\r
- printf("unexpected event data of second fetch and add : type=%d cookie=%d original%d\n",\r
- (int)ext_event->type,\r
- (int)dto_event->user_cookie.as_64,\r
- (long)atomic_buf);\r
- exit(1);\r
- }\r
-\r
- sleep(1); /* wait for other side to complete fetch_add */\r
-\r
- if (server) {\r
- printf("Server got original data = 0x%llx, expected 0x200\n", *atomic_buf);\r
- printf("Client final result (on server) = 0x%llx, expected 0x30\n", *target);\r
-\r
- if (*atomic_buf != 0x200 || *target != 0x30) {\r
- printf("ERROR: Server FETCH_ADD\n");\r
- exit(1);\r
- }\r
- } else {\r
- printf("Server side original data = 0x%llx, expected 0x20\n", *atomic_buf);\r
- printf("Server final result (on client) = 0x%llx, expected 0x300\n", *target);\r
-\r
- if (*atomic_buf != 0x20 || *target != 0x300) {\r
- printf("ERROR: Server FETCH_ADD\n");\r
- exit(1);\r
- }\r
- }\r
- printf("\n FETCH_ADD test - PASSED\n");\r
- return(0);\r
-}\r
-\r
-int\r
-main(int argc, char **argv)\r
-{\r
- char *hostname;\r
-\r
- if (argc > 2) {\r
- printf(usage);\r
- exit(1);\r
- }\r
-\r
- if ((argc == 1) || strcmp(argv[ 1 ], "-s") == 0)\r
- {\r
- server = 1;\r
- } else {\r
- server = 0;\r
- hostname = argv[ 1 ];\r
- }\r
-\r
-\r
- /*\r
- * connect\r
- */\r
- if (connect_ep(hostname)) {\r
- exit(1);\r
- }\r
- if (do_immediate()) {\r
- exit(1);\r
- }\r
- if (do_cmp_swap()) {\r
- exit(1);\r
- }\r
- if (do_fetch_add()) {\r
- exit(1);\r
- }\r
- return (disconnect_ep());\r
-}\r
+
+ if (server) {
+ status = dat_psp_free(psp);
+ _OK(status, "dat_psp_free");
+ }
+
+ for (i = 0; i < REG_MEM_COUNT; i++) {
+ status = dat_lmr_free(lmr[ i ]);
+ _OK(status, "dat_lmr_free");
+ }
+
+ status = dat_lmr_free(lmr_atomic);
+ _OK(status, "dat_lmr_free_atomic");
+
+ status = dat_ep_free(ep);
+ _OK(status, "dat_ep_free");
+
+ status = dat_evd_free(dto_evd);
+ _OK(status, "dat_evd_free DTO");
+ status = dat_evd_free(con_evd);
+ _OK(status, "dat_evd_free CON");
+ status = dat_evd_free(cr_evd);
+ _OK(status, "dat_evd_free CR");
+
+ status = dat_pz_free(pz);
+ _OK(status, "dat_pz_free");
+
+ status = dat_ia_close(ia, DAT_CLOSE_DEFAULT);
+ _OK(status, "dat_ia_close");
+
+ return(0);
+}
+
+int
+do_immediate()
+{
+ DAT_REGION_DESCRIPTION region;
+ DAT_EVENT event;
+ DAT_COUNT nmore;
+ DAT_LMR_TRIPLET iov;
+ DAT_RMR_TRIPLET r_iov;
+ DAT_DTO_COOKIE cookie;
+ DAT_RMR_CONTEXT their_context;
+ DAT_RETURN status;
+ DAT_UINT32 immed_data;
+ DAT_UINT32 immed_data_recv;
+ DAT_DTO_COMPLETION_EVENT_DATA *dto_event =
+ &event.event_data.dto_completion_event_data;
+ DAT_IB_EXTENSION_EVENT_DATA *ext_event =
+ (DAT_IB_EXTENSION_EVENT_DATA *)&event.event_extension_data[0];
+
+ printf("\nDoing RDMA WRITE IMMEDIATE DATA\n");
+
+ if (server) {
+ immed_data = 0x1111;
+ } else {
+ immed_data = 0x7777;
+ }
+
+ cookie.as_64 = 0x5555;
+
+ r_iov = *buf[ RECV_BUF_INDEX ];
+
+ iov.lmr_context = lmr_context[ SND_RDMA_BUF_INDEX ];
+ iov.virtual_address = (DAT_VADDR)(unsigned long)buf[ SND_RDMA_BUF_INDEX ];
+ iov.segment_length = BUF_SIZE;
+
+ cookie.as_64 = 0x9999;
+
+ status = dat_ib_post_rdma_write_immed(ep, // ep_handle
+ 1, // num_segments
+ &iov, // LMR
+ cookie, // user_cookie
+ &r_iov, // RMR
+ immed_data,
+ DAT_COMPLETION_DEFAULT_FLAG);
+ _OK(status, "dat_ib_post_rdma_write_immed");
+
+ /*
+ * Collect first event, write completion or the inbound recv with immed
+ */
+ status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
+ _OK(status, "dat_evd_wait after dat_ib_post_rdma_write");
+ if (event.event_number != DAT_IB_DTO_EVENT)
+ {
+ printf("unexpected event # waiting for WR-IMMED - 0x%x\n",
+ event.event_number);
+ exit(1);
+ }
+
+ _OK(dto_event->status, "event status");
+ if (ext_event->type == DAT_IB_RDMA_WRITE_IMMED)
+ {
+ if ((dto_event->transfered_length != BUF_SIZE) ||
+ (dto_event->user_cookie.as_64 != 0x9999))
+ {
+ printf("unexpected event data for rdma_write_immed: len=%d cookie=0x%x\n",
+ (int)dto_event->transfered_length,
+ (int)dto_event->user_cookie.as_64);
+ exit(1);
+ }
+ }
+ else if (ext_event->type == DAT_IB_RDMA_WRITE_IMMED_DATA)
+ {
+ if ((dto_event->transfered_length != BUF_SIZE) ||
+ (dto_event->user_cookie.as_64 != RECV_BUF_INDEX+1))
+ {
+ printf("unexpected event data of immediate write: len=%d cookie=%d expected %d/%d\n",
+ (int)dto_event->transfered_length,
+ (int)dto_event->user_cookie.as_64,
+ sizeof(int), RECV_BUF_INDEX+1);
+ exit(1);
+ }
+
+ /* get immediate data from event */
+ immed_data_recv = ext_event->val.immed.data;
+ }
+ else
+ {
+ printf("unexpected extension type for event - 0x%x, 0x%x\n",
+ event.event_number, ext_event->type);
+ exit(1);
+ }
+
+
+ /*
+ * Collect second event, write completion or the inbound recv with immed
+ */
+ status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
+ _OK(status, "dat_evd_wait after dat_ib_post_rdma_write");
+ if (event.event_number != DAT_IB_DTO_EVENT)
+ {
+ printf("unexpected event # waiting for WR-IMMED - 0x%x\n",
+ event.event_number);
+ exit(1);
+ }
+
+ _OK(dto_event->status, "event status");
+ if (ext_event->type == DAT_IB_RDMA_WRITE_IMMED)
+ {
+ if ((dto_event->transfered_length != BUF_SIZE) ||
+ (dto_event->user_cookie.as_64 != 0x9999))
+ {
+ printf("unexpected event data for rdma_write_immed: len=%d cookie=0x%x\n",
+ (int)dto_event->transfered_length,
+ (int)dto_event->user_cookie.as_64);
+ exit(1);
+ }
+ }
+ else if (ext_event->type == DAT_IB_RDMA_WRITE_IMMED_DATA)
+ {
+ if ((dto_event->transfered_length != BUF_SIZE) ||
+ (dto_event->user_cookie.as_64 != RECV_BUF_INDEX+1))
+ {
+ printf("unexpected event data of immediate write: len=%d cookie=%d expected %d/%d\n",
+ (int)dto_event->transfered_length,
+ (int)dto_event->user_cookie.as_64,
+ sizeof(int), RECV_BUF_INDEX+1);
+ exit(1);
+ }
+
+ /* get immediate data from event */
+ immed_data_recv = ext_event->val.immed.data;
+ }
+ else
+ {
+ printf("unexpected extension type for event - 0x%x, 0x%x\n",
+ event.event_number, ext_event->type);
+ exit(1);
+ }
+
+ if ((server) && (immed_data_recv != 0x7777))
+ {
+ printf("ERROR: Server got unexpected immed_data_recv 0x%x/0x%x\n",
+ 0x7777, immed_data_recv);
+ exit(1);
+ }
+ else if ((!server) && (immed_data_recv != 0x1111))
+ {
+ printf("ERROR: Client got unexpected immed_data_recv 0x%x/0x%x\n",
+ 0x1111, immed_data_recv);
+ exit(1);
+ }
+
+ if (server)
+ printf("Server received immed_data=0x%x\n", immed_data_recv);
+ else
+ printf("Client received immed_data=0x%x\n", immed_data_recv);
+
+ printf("rdma buffer %p contains: %s\n",
+ buf[ RCV_RDMA_BUF_INDEX ], buf[ RCV_RDMA_BUF_INDEX ]);
+
+ printf("\n RDMA_WRITE_WITH_IMMEDIATE_DATA test - PASSED\n");
+ return (0);
+}
+
+int
+do_cmp_swap()
+{
+ DAT_DTO_COOKIE cookie;
+ DAT_RETURN status;
+ DAT_EVENT event;
+ DAT_COUNT nmore;
+ DAT_LMR_TRIPLET l_iov;
+ DAT_RMR_TRIPLET r_iov;
+ volatile DAT_UINT64 *target = (DAT_UINT64*)buf[ RCV_RDMA_BUF_INDEX ];
+ DAT_DTO_COMPLETION_EVENT_DATA *dto_event =
+ &event.event_data.dto_completion_event_data;
+ DAT_IB_EXTENSION_EVENT_DATA *ext_event =
+ (DAT_IB_EXTENSION_EVENT_DATA *)&event.event_extension_data[0];
+
+ printf("\nDoing CMP and SWAP\n");
+
+ r_iov = *buf[ RECV_BUF_INDEX ];
+
+ l_iov.lmr_context = lmr_atomic_context;
+ l_iov.virtual_address = (DAT_VADDR)(unsigned long)atomic_buf;
+ l_iov.segment_length = BUF_SIZE_ATOMIC;
+
+ cookie.as_64 = 3333;
+ if (server) {
+ *target = 0x12345;
+ sleep(1);
+ /* server does not compare and should not swap */
+ status = dat_ib_post_cmp_and_swap( ep,
+ (DAT_UINT64)0x654321,
+ (DAT_UINT64)0x6789A,
+ &l_iov,
+ cookie,
+ &r_iov,
+ DAT_COMPLETION_DEFAULT_FLAG);
+ } else {
+ *target = 0x54321;
+ sleep(1);
+ /* client does compare and should swap */
+ status = dat_ib_post_cmp_and_swap( ep,
+ (DAT_UINT64)0x12345,
+ (DAT_UINT64)0x98765,
+ &l_iov,
+ cookie,
+ &r_iov,
+ DAT_COMPLETION_DEFAULT_FLAG);
+ }
+ _OK(status, "dat_ib_post_cmp_and_swap");
+ status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
+ _OK(status, "dat_evd_wait for compare and swap");
+ if (event.event_number != DAT_IB_DTO_EVENT) {
+ printf("unexpected event after post_cmp_and_swap: 0x%x\n",
+ event.event_number);
+ exit(1);
+ }
+
+ _OK(dto_event->status, "event status for CMP and SWAP");
+ if (ext_event->type != DAT_IB_CMP_AND_SWAP) {
+ printf("unexpected event data of cmp and swap : type=%d cookie=%d original 0x%llx\n",
+ (int)ext_event->type,
+ (int)dto_event->user_cookie.as_64,
+ *atomic_buf);
+ exit(1);
+ }
+ sleep(1); /* wait for other side to complete swap */
+ if (server) {
+ printf("Server got original data = 0x%llx, expected 0x54321\n", *atomic_buf);
+ printf("Client final result (on server) = 0x%llx, expected 0x98765\n", *target);
+
+ if (*atomic_buf != 0x54321 || *target != 0x98765) {
+ printf("ERROR: Server CMP_SWAP\n");
+ exit(1);
+ }
+ } else {
+ printf("Client got original data = 0x%llx, expected 0x12345\n",*atomic_buf);
+ printf("Server final result (on client) = 0x%llx, expected 0x54321\n", *target);
+
+ if (*atomic_buf != 0x12345 || *target != 0x54321) {
+ printf("ERROR: Client CMP_SWAP\n");
+ exit(1);
+ }
+ }
+ printf("\n CMP_SWAP test - PASSED\n");
+ return(0);
+}
+
+int
+do_fetch_add()
+{
+ DAT_DTO_COOKIE cookie;
+ DAT_RETURN status;
+ DAT_EVENT event;
+ DAT_COUNT nmore;
+ DAT_LMR_TRIPLET l_iov;
+ DAT_RMR_TRIPLET r_iov;
+ volatile DAT_UINT64 *target = (DAT_UINT64*)buf[ RCV_RDMA_BUF_INDEX ];
+ DAT_DTO_COMPLETION_EVENT_DATA *dto_event =
+ &event.event_data.dto_completion_event_data;
+ DAT_IB_EXTENSION_EVENT_DATA *ext_event =
+ (DAT_IB_EXTENSION_EVENT_DATA *)&event.event_extension_data[0];
+
+ printf("\nDoing FETCH and ADD\n");
+
+ r_iov = *buf[ RECV_BUF_INDEX ];
+
+ l_iov.lmr_context = lmr_atomic_context;
+ l_iov.virtual_address = (DAT_VADDR)(unsigned long)atomic_buf;
+ l_iov.segment_length = BUF_SIZE_ATOMIC;
+
+ cookie.as_64 = 0x7777;
+ if (server) {
+ /* Wait for client to finish cmp_swap */
+ while (*target != 0x98765)
+ sleep(1);
+ *target = 0x10;
+ sleep(1);
+ status = dat_ib_post_fetch_and_add( ep,
+ (DAT_UINT64)0x100,
+ &l_iov,
+ cookie,
+ &r_iov,
+ DAT_COMPLETION_DEFAULT_FLAG);
+ } else {
+ /* Wait for server, no swap so nothing to check */
+ *target = 0x100;
+ sleep(1);
+ status = dat_ib_post_fetch_and_add( ep,
+ (DAT_UINT64)0x10,
+ &l_iov,
+ cookie,
+ &r_iov,
+ DAT_COMPLETION_DEFAULT_FLAG);
+ }
+ _OK(status, "dat_ib_post_fetch_and_add");
+ status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
+ _OK(status, "dat_evd_wait for fetch and add");
+ if (event.event_number != DAT_IB_DTO_EVENT) {
+ printf("unexpected event after post_fetch_and_add: 0x%x\n", event.event_number);
+ exit(1);
+ }
+
+ _OK(dto_event->status, "event status for FETCH and ADD");
+ if (ext_event->type != DAT_IB_FETCH_AND_ADD) {
+ printf("unexpected event data of fetch and add : type=%d cookie=%d original%d\n",
+ (int)ext_event->type,
+ (int)dto_event->user_cookie.as_64,
+ (int)*atomic_buf);
+ exit(1);
+ }
+
+ if (server) {
+ printf("Client original data (on server) = 0x%llx, expected 0x100\n", *atomic_buf);
+ } else {
+ printf("Server original data (on client) = 0x%llx, expected 0x10\n", *atomic_buf);
+ }
+
+ sleep(1);
+
+ if (server) {
+ status = dat_ib_post_fetch_and_add( ep,
+ (DAT_UINT64)0x100,
+ &l_iov,
+ cookie,
+ &r_iov,
+ DAT_COMPLETION_DEFAULT_FLAG);
+ } else {
+ status = dat_ib_post_fetch_and_add( ep,
+ (DAT_UINT64)0x10,
+ &l_iov,
+ cookie,
+ &r_iov,
+ DAT_COMPLETION_DEFAULT_FLAG);
+ }
+
+ status = dat_evd_wait(dto_evd, DTO_TIMEOUT, 1, &event, &nmore);
+ _OK(status, "dat_evd_wait for second fetch and add");
+ if (event.event_number != DAT_IB_DTO_EVENT) {
+ printf("unexpected event after second post_fetch_and_add: 0x%x\n", event.event_number);
+ exit(1);
+ }
+
+ _OK(dto_event->status, "event status for second FETCH and ADD");
+ if (ext_event->type != DAT_IB_FETCH_AND_ADD) {
+ printf("unexpected event data of second fetch and add : type=%d cookie=%d original%d\n",
+ (int)ext_event->type,
+ (int)dto_event->user_cookie.as_64,
+ (long)atomic_buf);
+ exit(1);
+ }
+
+ sleep(1); /* wait for other side to complete fetch_add */
+
+ if (server) {
+ printf("Server got original data = 0x%llx, expected 0x200\n", *atomic_buf);
+ printf("Client final result (on server) = 0x%llx, expected 0x30\n", *target);
+
+ if (*atomic_buf != 0x200 || *target != 0x30) {
+ printf("ERROR: Server FETCH_ADD\n");
+ exit(1);
+ }
+ } else {
+ printf("Server side original data = 0x%llx, expected 0x20\n", *atomic_buf);
+ printf("Server final result (on client) = 0x%llx, expected 0x300\n", *target);
+
+ if (*atomic_buf != 0x20 || *target != 0x300) {
+ printf("ERROR: Server FETCH_ADD\n");
+ exit(1);
+ }
+ }
+ printf("\n FETCH_ADD test - PASSED\n");
+ return(0);
+}
+
+int
+main(int argc, char **argv)
+{
+ char *hostname;
+
+ if (argc > 2) {
+ printf(usage);
+ exit(1);
+ }
+
+ if ((argc == 1) || strcmp(argv[ 1 ], "-s") == 0)
+ {
+ server = 1;
+ } else {
+ server = 0;
+ hostname = argv[ 1 ];
+ }
+
+
+ /*
+ * connect
+ */
+ if (connect_ep(hostname)) {
+ exit(1);
+ }
+ if (do_immediate()) {
+ exit(1);
+ }
+ if (do_cmp_swap()) {
+ exit(1);
+ }
+ if (do_fetch_add()) {
+ exit(1);
+ }
+ return (disconnect_ep());
+}
projects / ~ardavis / dapl.git / commitdiff