DIRS = \\r
- cmatose\r
+ cmatose \\r
+ rdma_server \\r
+ rdma_client\r
--- /dev/null
+TARGETNAME = rdma_client\r
+TARGETPATH = ..\..\..\..\bin\user\obj$(BUILD_ALT_DIR)\r
+TARGETTYPE = PROGRAM\r
+\r
+UMTYPE = console\r
+UMENTRY = main\r
+\r
+USE_MSVCRT = 1\r
+\r
+SOURCES = rdma_client.c\r
+ \r
+INCLUDES = ..;..\..\include;..\..\..\..\inc;..\..\..\..\inc\user;\\r
+ ..\..\..\libibverbs\include;..\..\..\..\inc\user\linux;\r
+\r
+TARGETLIBS = \\r
+ $(SDK_LIB_PATH)\kernel32.lib \\r
+ $(SDK_LIB_PATH)\advapi32.lib \\r
+ $(SDK_LIB_PATH)\user32.lib \\r
+ $(SDK_LIB_PATH)\ole32.lib \\r
+ $(SDK_LIB_PATH)\ws2_32.lib \\r
+!if $(FREEBUILD)\r
+ $(TARGETPATH)\*\libibverbs.lib \\r
+ $(TARGETPATH)\*\librdmacm.lib\r
+!else\r
+ $(TARGETPATH)\*\libibverbsd.lib \\r
+ $(TARGETPATH)\*\librdmacmd.lib\r
+!endif\r
+\r
--- /dev/null
+#\r
+# DO NOT EDIT THIS FILE!!! Edit .\sources. if you want to add a new source\r
+# file to this component. This file merely indirects to the real make file\r
+# that is shared by all the driver components of the OpenIB Windows project.\r
+#\r
+\r
+!INCLUDE ..\..\..\..\inc\openib.def\r
--- /dev/null
+/*
+ * Copyright (c) 2010 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under the OpenIB.org BSD license
+ * below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <stdio.h>\r
+#include <stdlib.h>\r
+#include <string.h>\r
+\r
+#include "..\..\..\..\etc\user\getopt.c"\r
+#include <rdma/rdma_verbs.h>\r
+#include <rdma/rdma_cma.h>\r
+
+static char *server = "127.0.0.1";
+static char *port = "7471";
+
+struct rdma_cm_id *id;
+struct ibv_mr *mr;
+uint8_t send_msg[16];
+uint8_t recv_msg[16];
+
+static int run(void)
+{
+ struct rdma_addrinfo hints, *res;
+ struct ibv_qp_init_attr attr;
+ struct ibv_wc wc;
+ int ret;
+
+ memset(&hints, 0, sizeof hints);
+ hints.ai_port_space = RDMA_PS_TCP;
+ ret = rdma_getaddrinfo(server, port, &hints, &res);
+ if (ret) {
+ printf("rdma_getaddrinfo %d\n", errno);
+ return ret;
+ }
+
+ memset(&attr, 0, sizeof attr);
+ attr.cap.max_send_wr = attr.cap.max_recv_wr = 1;
+ attr.cap.max_send_sge = attr.cap.max_recv_sge = 1;
+ attr.cap.max_inline_data = 16;
+ attr.qp_context = id;
+ attr.sq_sig_all = 1;
+ ret = rdma_create_ep(&id, res, NULL, &attr);
+ rdma_freeaddrinfo(res);
+ if (ret) {
+ printf("rdma_create_ep %d\n", errno);
+ return ret;
+ }
+
+ mr = rdma_reg_msgs(id, recv_msg, 16);
+ if (!mr) {
+ printf("rdma_reg_msgs %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_post_recv(id, NULL, recv_msg, 16, mr);
+ if (ret) {
+ printf("rdma_post_recv %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_connect(id, NULL);
+ if (ret) {
+ printf("rdma_connect %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_post_send(id, NULL, send_msg, 16, NULL, IBV_SEND_INLINE);
+ if (ret) {
+ printf("rdma_post_send %d\n", errno);
+ return ret;
+ }
+
+ ret = rdma_get_recv_comp(id, &wc);
+ if (ret <= 0) {
+ printf("rdma_get_recv_comp %d\n", ret);
+ return ret;
+ }
+
+ rdma_disconnect(id);
+ rdma_dereg_mr(mr);
+ rdma_destroy_ep(id);
+ return 0;
+}
+
+int main(int argc, char **argv)
+{
+ int op, ret;
+
+ while ((op = getopt(argc, argv, "s:p:")) != -1) {
+ switch (op) {
+ case 's':
+ server = optarg;
+ break;
+ case 'p':
+ port = optarg;
+ break;
+ default:
+ printf("usage: %s\n", argv[0]);
+ printf("\t[-s server_address]\n");
+ printf("\t[-p port_number]\n");
+ exit(1);
+ }
+ }
+
+ printf("rdma_client: start\n");
+ ret = run();
+ printf("rdma_client: end %d\n", ret);
+ return ret;
+}
--- /dev/null
+TARGETNAME = rdma_server\r
+TARGETPATH = ..\..\..\..\bin\user\obj$(BUILD_ALT_DIR)\r
+TARGETTYPE = PROGRAM\r
+\r
+UMTYPE = console\r
+UMENTRY = main\r
+\r
+USE_MSVCRT = 1\r
+\r
+SOURCES = rdma_server.c\r
+ \r
+INCLUDES = ..;..\..\include;..\..\..\..\inc;..\..\..\..\inc\user;\\r
+ ..\..\..\libibverbs\include;..\..\..\..\inc\user\linux;\r
+\r
+TARGETLIBS = \\r
+ $(SDK_LIB_PATH)\kernel32.lib \\r
+ $(SDK_LIB_PATH)\advapi32.lib \\r
+ $(SDK_LIB_PATH)\user32.lib \\r
+ $(SDK_LIB_PATH)\ole32.lib \\r
+ $(SDK_LIB_PATH)\ws2_32.lib \\r
+!if $(FREEBUILD)\r
+ $(TARGETPATH)\*\libibverbs.lib \\r
+ $(TARGETPATH)\*\librdmacm.lib\r
+!else\r
+ $(TARGETPATH)\*\libibverbsd.lib \\r
+ $(TARGETPATH)\*\librdmacmd.lib\r
+!endif\r
+\r
--- /dev/null
+#\r
+# DO NOT EDIT THIS FILE!!! Edit .\sources. if you want to add a new source\r
+# file to this component. This file merely indirects to the real make file\r
+# that is shared by all the driver components of the OpenIB Windows project.\r
+#\r
+\r
+!INCLUDE ..\..\..\..\inc\openib.def\r
--- /dev/null
+/*\r
+ * Copyright (c) 2005-2010 Intel Corporation. All rights reserved.\r
+ *\r
+ * This software is available to you under the OpenIB.org BSD license\r
+ * below:\r
+ *\r
+ * Redistribution and use in source and binary forms, with or\r
+ * without modification, are permitted provided that the following\r
+ * conditions are met:\r
+ *\r
+ * - Redistributions of source code must retain the above\r
+ * copyright notice, this list of conditions and the following\r
+ * disclaimer.\r
+ *\r
+ * - Redistributions in binary form must reproduce the above\r
+ * copyright notice, this list of conditions and the following\r
+ * disclaimer in the documentation and/or other materials\r
+ * provided with the distribution.\r
+ *\r
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,\r
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\r
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AWV\r
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS\r
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN\r
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\r
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r
+ * SOFTWARE.\r
+ */\r
+\r
+#include <stdio.h>\r
+#include <stdlib.h>\r
+#include <string.h>\r
+\r
+#include "..\..\..\..\etc\user\getopt.c"\r
+#include <rdma/rdma_verbs.h>\r
+#include <rdma/rdma_cma.h>\r
+\r
+static char *port = "7471";\r
+\r
+struct rdma_cm_id *listen_id, *id;\r
+struct ibv_mr *mr;\r
+uint8_t send_msg[16];\r
+uint8_t recv_msg[16];\r
+\r
+static int run(void)\r
+{\r
+ struct rdma_addrinfo hints, *res;\r
+ struct ibv_qp_init_attr attr;\r
+ struct ibv_wc wc;\r
+ int ret;\r
+\r
+ memset(&hints, 0, sizeof hints);\r
+ hints.ai_flags = RAI_PASSIVE;\r
+ hints.ai_port_space = RDMA_PS_TCP;\r
+ ret = rdma_getaddrinfo(NULL, port, &hints, &res);\r
+ if (ret) {\r
+ printf("rdma_getaddrinfo %d\n", errno);\r
+ return ret;\r
+ }\r
+\r
+ memset(&attr, 0, sizeof attr);\r
+ attr.cap.max_send_wr = attr.cap.max_recv_wr = 1;\r
+ attr.cap.max_send_sge = attr.cap.max_recv_sge = 1;\r
+ attr.cap.max_inline_data = 16;\r
+ attr.sq_sig_all = 1;\r
+ ret = rdma_create_ep(&listen_id, res, NULL, &attr);\r
+ rdma_freeaddrinfo(res);\r
+ if (ret) {\r
+ printf("rdma_create_ep %d\n", errno);\r
+ return ret;\r
+ }\r
+\r
+ ret = rdma_listen(listen_id, 0);\r
+ if (ret) {\r
+ printf("rdma_listen %d\n", errno);\r
+ return ret;\r
+ }\r
+\r
+ ret = rdma_get_request(listen_id, &id);\r
+ if (ret) {\r
+ printf("rdma_get_request %d\n", errno);\r
+ return ret;\r
+ }\r
+\r
+ mr = rdma_reg_msgs(id, recv_msg, 16);\r
+ if (!mr) {\r
+ printf("rdma_reg_msgs %d\n", errno);\r
+ return ret;\r
+ }\r
+\r
+ ret = rdma_post_recv(id, NULL, recv_msg, 16, mr);\r
+ if (ret) {\r
+ printf("rdma_post_recv %d\n", errno);\r
+ return ret;\r
+ }\r
+\r
+ ret = rdma_accept(id, NULL);\r
+ if (ret) {\r
+ printf("rdma_connect %d\n", errno);\r
+ return ret;\r
+ }\r
+\r
+ ret = rdma_get_recv_comp(id, &wc);\r
+ if (ret <= 0) {\r
+ printf("rdma_get_recv_comp %d\n", ret);\r
+ return ret;\r
+ }\r
+\r
+ ret = rdma_post_send(id, NULL, send_msg, 16, NULL, IBV_SEND_INLINE);\r
+ if (ret) {\r
+ printf("rdma_post_send %d\n", errno);\r
+ return ret;\r
+ }\r
+\r
+ ret = rdma_get_send_comp(id, &wc);\r
+ if (ret <= 0) {\r
+ printf("rdma_get_send_comp %d\n", ret);\r
+ return ret;\r
+ }\r
+\r
+ rdma_disconnect(id);\r
+ rdma_dereg_mr(mr);\r
+ rdma_destroy_ep(id);\r
+ rdma_destroy_ep(listen_id);\r
+ return 0;\r
+}\r
+\r
+int main(int argc, char **argv)\r
+{\r
+ int op, ret;\r
+\r
+ while ((op = getopt(argc, argv, "p:")) != -1) {\r
+ switch (op) {\r
+ case 'p':\r
+ port = optarg;\r
+ break;\r
+ default:\r
+ printf("usage: %s\n", argv[0]);\r
+ printf("\t[-p port_number]\n");\r
+ exit(1);\r
+ }\r
+ }\r
+\r
+ printf("rdma_server: start\n");\r
+ ret = run();\r
+ printf("rdma_server: end %d\n", ret);\r
+ return ret;\r
+}\r
--- /dev/null
+/*
+ * Copyright (c) 2010 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#if !defined(RDMA_VERBS_H)
+#define RDMA_VERBS_H
+
+#include <assert.h>
+#include <infiniband/verbs.h>
+#include <rdma/rdma_cma.h>
+#include <errno.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+static __inline int rdma_seterrno(int ret)
+{
+ if (ret) {
+ errno = ret;
+ ret = -1;
+ }
+ return ret;
+}
+
+/*
+ * Memory registration helpers.
+ */
+static __inline struct ibv_mr *
+rdma_reg_msgs(struct rdma_cm_id *id, void *addr, size_t length)
+{
+ return ibv_reg_mr(id->qp->pd, addr, length, IBV_ACCESS_LOCAL_WRITE);
+}
+
+static __inline struct ibv_mr *
+rdma_reg_read(struct rdma_cm_id *id, void *addr, size_t length)
+{
+ return ibv_reg_mr(id->qp->pd, addr, length, IBV_ACCESS_LOCAL_WRITE |
+ IBV_ACCESS_REMOTE_READ);
+}
+
+static __inline struct ibv_mr *
+rdma_reg_write(struct rdma_cm_id *id, void *addr, size_t length)
+{
+ return ibv_reg_mr(id->qp->pd, addr, length, IBV_ACCESS_LOCAL_WRITE |
+ IBV_ACCESS_REMOTE_WRITE);
+}
+
+static __inline int
+rdma_dereg_mr(struct ibv_mr *mr)
+{
+ return rdma_seterrno(ibv_dereg_mr(mr));
+}
+
+
+/*
+ * Vectored send, receive, and RDMA operations.
+ * Support multiple scatter-gather entries.
+ */
+static __inline int
+rdma_post_recvv(struct rdma_cm_id *id, void *context, struct ibv_sge *sgl,
+ int nsge)
+{
+ struct ibv_recv_wr wr, *bad;
+
+ wr.wr_id = (uintptr_t) context;
+ wr.next = NULL;
+ wr.sg_list = sgl;
+ wr.num_sge = nsge;
+
+ return rdma_seterrno(ibv_post_recv(id->qp, &wr, &bad));
+}
+
+static __inline int
+rdma_post_sendv(struct rdma_cm_id *id, void *context, struct ibv_sge *sgl,
+ int nsge, int flags)
+{
+ struct ibv_send_wr wr, *bad;
+
+ wr.wr_id = (uintptr_t) context;
+ wr.next = NULL;
+ wr.sg_list = sgl;
+ wr.num_sge = nsge;
+ wr.opcode = IBV_WR_SEND;
+ wr.send_flags = flags;
+
+ return rdma_seterrno(ibv_post_send(id->qp, &wr, &bad));
+}
+
+static __inline int
+rdma_post_readv(struct rdma_cm_id *id, void *context, struct ibv_sge *sgl,
+ int nsge, int flags, uint64_t remote_addr, uint32_t rkey)
+{
+ struct ibv_send_wr wr, *bad;
+
+ wr.wr_id = (uintptr_t) context;
+ wr.next = NULL;
+ wr.sg_list = sgl;
+ wr.num_sge = nsge;
+ wr.opcode = IBV_WR_RDMA_READ;
+ wr.send_flags = flags;
+ wr.wr.rdma.remote_addr = remote_addr;
+ wr.wr.rdma.rkey = rkey;
+
+ return rdma_seterrno(ibv_post_send(id->qp, &wr, &bad));
+}
+
+static __inline int
+rdma_post_writev(struct rdma_cm_id *id, void *context, struct ibv_sge *sgl,
+ int nsge, int flags, uint64_t remote_addr, uint32_t rkey)
+{
+ struct ibv_send_wr wr, *bad;
+
+ wr.wr_id = (uintptr_t) context;
+ wr.next = NULL;
+ wr.sg_list = sgl;
+ wr.num_sge = nsge;
+ wr.opcode = IBV_WR_RDMA_WRITE;
+ wr.send_flags = flags;
+ wr.wr.rdma.remote_addr = remote_addr;
+ wr.wr.rdma.rkey = rkey;
+
+ return rdma_seterrno(ibv_post_send(id->qp, &wr, &bad));
+}
+
+/*
+ * Simple send, receive, and RDMA calls.
+ */
+static __inline int
+rdma_post_recv(struct rdma_cm_id *id, void *context, void *addr,
+ size_t length, struct ibv_mr *mr)
+{
+ struct ibv_sge sge;
+
+ assert((addr >= mr->addr) && ((addr + length) <= (mr->addr + mr->length)));
+ sge.addr = (uint64_t) (uintptr_t) addr;
+ sge.length = (uint32_t) length;
+ sge.lkey = mr->lkey;
+
+ return rdma_post_recvv(id, context, &sge, 1);
+}
+
+static __inline int
+rdma_post_send(struct rdma_cm_id *id, void *context, void *addr,
+ size_t length, struct ibv_mr *mr, int flags)
+{
+ struct ibv_sge sge;
+
+ sge.addr = (uint64_t) (uintptr_t) addr;
+ sge.length = (uint32_t) length;
+ sge.lkey = mr ? mr->lkey : 0;
+
+ return rdma_post_sendv(id, context, &sge, 1, flags);
+}
+
+static __inline int
+rdma_post_read(struct rdma_cm_id *id, void *context, void *addr,
+ size_t length, struct ibv_mr *mr, int flags,
+ uint64_t remote_addr, uint32_t rkey)
+{
+ struct ibv_sge sge;
+
+ sge.addr = (uint64_t) (uintptr_t) addr;
+ sge.length = (uint32_t) length;
+ sge.lkey = mr->lkey;
+
+ return rdma_post_readv(id, context, &sge, 1, flags, remote_addr, rkey);
+}
+
+static __inline int
+rdma_post_write(struct rdma_cm_id *id, void *context, void *addr,
+ size_t length, struct ibv_mr *mr, int flags,
+ uint64_t remote_addr, uint32_t rkey)
+{
+ struct ibv_sge sge;
+
+ sge.addr = (uint64_t) (uintptr_t) addr;
+ sge.length = (uint32_t) length;
+ sge.lkey = mr ? mr->lkey : 0;
+
+ return rdma_post_writev(id, context, &sge, 1, flags, remote_addr, rkey);
+}
+
+static __inline int
+rdma_post_ud_send(struct rdma_cm_id *id, void *context, void *addr,
+ size_t length, struct ibv_mr *mr, int flags,
+ struct ibv_ah *ah, uint32_t remote_qpn)
+{
+ struct ibv_send_wr wr, *bad;
+ struct ibv_sge sge;
+
+ sge.addr = (uint64_t) (uintptr_t) addr;
+ sge.length = (uint32_t) length;
+ sge.lkey = mr ? mr->lkey : 0;
+
+ wr.wr_id = (uintptr_t) context;
+ wr.next = NULL;
+ wr.sg_list = &sge;
+ wr.num_sge = 1;
+ wr.opcode = IBV_WR_SEND;
+ wr.send_flags = flags;
+ wr.wr.ud.ah = ah;
+ wr.wr.ud.remote_qpn = remote_qpn;
+ wr.wr.ud.remote_qkey = RDMA_UDP_QKEY;
+
+ return rdma_seterrno(ibv_post_send(id->qp, &wr, &bad));
+}
+
+// Comment out until patch to automatically create CQs
+static __inline int
+rdma_get_send_comp(struct rdma_cm_id *id, struct ibv_wc *wc)
+{
+ struct ibv_cq *cq;
+ void *context;
+ int ret;
+
+ ret = ibv_poll_cq(id->send_cq, 1, wc);
+ if (ret)
+ goto out;
+
+ ret = ibv_req_notify_cq(id->send_cq, 0);
+ if (ret)
+ return rdma_seterrno(ret);
+
+ while (!(ret = ibv_poll_cq(id->send_cq, 1, wc))) {
+ ret = ibv_get_cq_event(id->send_cq_channel, &cq, &context);
+ if (ret)
+ return rdma_seterrno(ret);
+
+ assert(cq == id->send_cq && context == id);
+ ibv_ack_cq_events(id->send_cq, 1);
+ }
+out:
+ return (ret < 0) ? rdma_seterrno(ret) : ret;
+}
+
+static __inline int
+rdma_get_recv_comp(struct rdma_cm_id *id, struct ibv_wc *wc)
+{
+ struct ibv_cq *cq;
+ void *context;
+ int ret;
+
+ ret = ibv_poll_cq(id->recv_cq, 1, wc);
+ if (ret)
+ goto out;
+
+ ret = ibv_req_notify_cq(id->recv_cq, 0);
+ if (ret)
+ return rdma_seterrno(ret);
+
+ while (!(ret = ibv_poll_cq(id->recv_cq, 1, wc))) {
+ ret = ibv_get_cq_event(id->recv_cq_channel, &cq, &context);
+ if (ret)
+ return rdma_seterrno(ret);
+
+ assert(cq == id->recv_cq && context == id);
+ ibv_ack_cq_events(id->recv_cq, 1);
+ }
+out:
+ return (ret < 0) ? rdma_seterrno(ret) : ret;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RDMA_CMA_H */
projects / ~shefty / rdma-win.git / commitdiff