--- /dev/null
+Sean Hefty <sean.hefty@intel.com>
--- /dev/null
+This software is available to you under a choice of one of two
+licenses. You may choose to be licensed under the terms of the the
+OpenIB.org BSD license or the GNU General Public License (GPL) Version
+2, both included below.
+
+Copyright (c) 2005 Intel Corporation. All rights reserved.
+
+==================================================================
+
+ OpenIB.org BSD license
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following
+ disclaimer in the documentation and/or other materials provided
+ with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+==================================================================
+
+ GNU GENERAL PUBLIC LICENSE
+ Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.
+ 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+ Preamble
+
+ The licenses for most software are designed to take away your
+freedom to share and change it. By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users. This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it. (Some other Free Software Foundation software is covered by
+the GNU Library General Public License instead.) You can apply it to
+your programs, too.
+
+ When we speak of free software, we are referring to freedom, not
+price. Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+ To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+ For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have. You must make sure that they, too, receive or can get the
+source code. And you must show them these terms so they know their
+rights.
+
+ We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+ Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software. If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+ Finally, any free program is threatened constantly by software
+patents. We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary. To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+ The precise terms and conditions for copying, distribution and
+modification follow.
+
+ GNU GENERAL PUBLIC LICENSE
+ TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+ 0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License. The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language. (Hereinafter, translation is included without limitation in
+the term "modification".) Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope. The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+ 1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+ 2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+ a) You must cause the modified files to carry prominent notices
+ stating that you changed the files and the date of any change.
+
+ b) You must cause any work that you distribute or publish, that in
+ whole or in part contains or is derived from the Program or any
+ part thereof, to be licensed as a whole at no charge to all third
+ parties under the terms of this License.
+
+ c) If the modified program normally reads commands interactively
+ when run, you must cause it, when started running for such
+ interactive use in the most ordinary way, to print or display an
+ announcement including an appropriate copyright notice and a
+ notice that there is no warranty (or else, saying that you provide
+ a warranty) and that users may redistribute the program under
+ these conditions, and telling the user how to view a copy of this
+ License. (Exception: if the Program itself is interactive but
+ does not normally print such an announcement, your work based on
+ the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole. If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works. But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+ 3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+ a) Accompany it with the complete corresponding machine-readable
+ source code, which must be distributed under the terms of Sections
+ 1 and 2 above on a medium customarily used for software interchange; or,
+
+ b) Accompany it with a written offer, valid for at least three
+ years, to give any third party, for a charge no more than your
+ cost of physically performing source distribution, a complete
+ machine-readable copy of the corresponding source code, to be
+ distributed under the terms of Sections 1 and 2 above on a medium
+ customarily used for software interchange; or,
+
+ c) Accompany it with the information you received as to the offer
+ to distribute corresponding source code. (This alternative is
+ allowed only for noncommercial distribution and only if you
+ received the program in object code or executable form with such
+ an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it. For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable. However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+ 4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License. Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+ 5. You are not required to accept this License, since you have not
+signed it. However, nothing else grants you permission to modify or
+distribute the Program or its derivative works. These actions are
+prohibited by law if you do not accept this License. Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+ 6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions. You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+ 7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License. If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all. For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices. Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+ 8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded. In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+ 9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time. Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number. If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation. If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+ 10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission. For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this. Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+ NO WARRANTY
+
+ 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+ 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+ END OF TERMS AND CONDITIONS
+
+ How to Apply These Terms to Your New Programs
+
+ If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+ To do so, attach the following notices to the program. It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+ <one line to give the program's name and a brief idea of what it does.>
+ Copyright (C) <year> <name of author>
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+ Gnomovision version 69, Copyright (C) year name of author
+ Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+ This is free software, and you are welcome to redistribute it
+ under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License. Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary. Here is a sample; alter the names:
+
+ Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+ `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+ <signature of Ty Coon>, 1 April 1989
+ Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs. If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library. If this is what you want to do, use the GNU Library General
+Public License instead of this License.
--- /dev/null
+# $Id: Makefile.am 3373 2005-09-12 16:34:20Z roland $
+INCLUDES = -I$(srcdir)/include
+
+AM_CFLAGS = -g -Wall -D_GNU_SOURCE
+
+rdmacmlibdir = $(libdir)
+
+rdmacmlib_LTLIBRARIES = src/librdmacm.la
+
+src_rdmacm_la_CFLAGS = -g -Wall -D_GNU_SOURCE
+
+if HAVE_LD_VERSION_SCRIPT
+ rdmacm_version_script = -Wl,--version-script=$(srcdir)/src/librdmacm.map
+else
+ rdmacm_version_script =
+endif
+
+src_librdmacm_la_SOURCES = src/cma.c
+src_librdmacm_la_LDFLAGS = -avoid-version $(rdmacm_version_script)
+
+bin_PROGRAMS = examples/ucmatose
+examples_ucmatose_SOURCES = examples/cmatose.c
+examples_ucmatose_LDADD = $(top_builddir)/src/librdmacm.la
+
+librdmacmincludedir = $(includedir)/rdma
+
+librdmacminclude_HEADERS = include/rdma/rdma_cma_abi.h \
+ include/rdma/rdma_cma.h
+
+EXTRA_DIST = include/rdma/rdma_cma_abi.h \
+ include/rdma/rdma_cma.h \
+ src/librdmacm.map \
+ librdmacm.spec.in
+
+dist-hook: librdmacm.spec
+ cp librdmacm.spec $(distdir)
--- /dev/null
+This README is for userspace RDMA cm library.
+
+Building
+
+To make this directory, run:
+./autogen.sh && ./configure && make && make install
+
+Typically the autogen and configure steps only need be done the first
+time unless configure.in or Makefile.am changes.
+
+Libraries are installed by default at /usr/local/lib.
+
+Device files
+
+The userspace CMA uses a single device file regardless of the number
+of adapters or ports present.
+
+To create the appropriate character device file automatically with
+udev, a rule like
+
+ KERNEL="ucma", NAME="infiniband/%k", MODE="0666"
+
+can be used. This will create the device node named
+
+ /dev/infiniband/ucma
+
+or you can create it manually
+
+ mknod /dev/infiniband/ucma c 231 255
--- /dev/null
+#! /bin/sh
+
+set -x
+aclocal -I config
+libtoolize --force --copy
+autoheader
+automake --foreign --add-missing --copy
+autoconf
--- /dev/null
+dnl Process this file with autoconf to produce a configure script.
+
+AC_PREREQ(2.57)
+AC_INIT(librdmacm, 0.9.0, openib-general@openib.org)
+AC_CONFIG_SRCDIR([src/cma.c])
+AC_CONFIG_AUX_DIR(config)
+AM_CONFIG_HEADER(config.h)
+AM_INIT_AUTOMAKE(librdmacm, 0.9.0)
+AC_DISABLE_STATIC
+AM_PROG_LIBTOOL
+
+AC_ARG_ENABLE(libcheck, [ --disable-libcheck do not test for presence of ib libraries],
+[ if test x$enableval = xno ; then
+ disable_libcheck=yes
+ fi
+])
+
+dnl Checks for programs
+AC_PROG_CC
+
+dnl Checks for typedefs, structures, and compiler characteristics.
+AC_C_CONST
+AC_CHECK_SIZEOF(long)
+
+dnl Checks for libraries
+if test "$disable_libcheck" != "yes"
+then
+AC_CHECK_LIB(ibverbs, ibv_get_devices, [],
+ AC_MSG_ERROR([ibv_get_devices() not found. librdmacm requires libibverbs.]))
+fi
+
+dnl Checks for header files.
+if test "$disable_libcheck" != "yes"
+then
+AC_CHECK_HEADER(infiniband/verbs.h, [],
+ AC_MSG_ERROR([<infiniband/verbs.h> not found. Is libibverbs installed?]))
+fi
+AC_HEADER_STDC
+
+AC_CACHE_CHECK(whether ld accepts --version-script, ac_cv_version_script,
+ if test -n "`$LD --help < /dev/null 2>/dev/null | grep version-script`"; then
+ ac_cv_version_script=yes
+ else
+ ac_cv_version_script=no
+ fi)
+
+AM_CONDITIONAL(HAVE_LD_VERSION_SCRIPT, test "$ac_cv_version_script" = "yes")
+
+AC_CONFIG_FILES([Makefile librdmacm.spec])
+AC_OUTPUT
--- /dev/null
+/*
+ * Copyright (c) 2005 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id$
+ */
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <netinet/in.h>
+#include <sys/socket.h>
+#include <netdb.h>
+#include <byteswap.h>
+
+#include <rdma/rdma_cma.h>
+
+#if __BYTE_ORDER == __BIG_ENDIAN
+static inline uint64_t cpu_to_be64(uint64_t x) { return x; }
+static inline uint32_t cpu_to_be32(uint32_t x) { return x; }
+#else
+static inline uint64_t cpu_to_be64(uint64_t x) { return bswap_64(x); }
+static inline uint32_t cpu_to_be32(uint32_t x) { return bswap_32(x); }
+#endif
+
+/*
+ * To execute:
+ * Server: rdma_cmatose
+ * Client: rdma_cmatose "dst_ip=ip"
+ */
+
+struct cmatest_node {
+ int id;
+ struct rdma_cm_id *cma_id;
+ int connected;
+ struct ibv_pd *pd;
+ struct ibv_cq *cq;
+ struct ibv_mr *mr;
+ void *mem;
+};
+
+struct cmatest {
+ struct cmatest_node *nodes;
+ int conn_index;
+ int connects_left;
+ int disconnects_left;
+
+ struct sockaddr_in addr_in;
+ struct sockaddr *addr;
+};
+
+static struct cmatest test;
+static int connections = 1;
+static int message_size = 100;
+static int message_count = 10;
+static int is_server;
+
+static int create_message(struct cmatest_node *node)
+{
+ if (!message_size)
+ message_count = 0;
+
+ if (!message_count)
+ return 0;
+
+ node->mem = malloc(message_size);
+ if (!node->mem) {
+ printf("failed message allocation\n");
+ return -1;
+ }
+ node->mr = ibv_reg_mr(node->pd, node->mem, message_size,
+ IBV_ACCESS_LOCAL_WRITE);
+ if (!node->mr) {
+ printf("failed to reg MR\n");
+ goto err;
+ }
+ return 0;
+err:
+ free(node->mem);
+ return -1;
+}
+
+static int init_node(struct cmatest_node *node)
+{
+ struct ibv_qp_init_attr init_qp_attr;
+ int cqe, ret;
+
+ node->pd = ibv_alloc_pd(node->cma_id->verbs);
+ if (!node->pd) {
+ ret = -ENOMEM;
+ printf("cmatose: unable to allocate PD\n");
+ goto out;
+ }
+
+ cqe = message_count ? message_count * 2 : 2;
+ node->cq = ibv_create_cq(node->cma_id->verbs, cqe, node, 0, 0);
+ if (!node->cq) {
+ ret = -ENOMEM;
+ printf("cmatose: unable to create CQ\n");
+ goto out;
+ }
+
+ memset(&init_qp_attr, 0, sizeof init_qp_attr);
+ init_qp_attr.cap.max_send_wr = message_count ? message_count : 1;
+ init_qp_attr.cap.max_recv_wr = message_count ? message_count : 1;
+ init_qp_attr.cap.max_send_sge = 1;
+ init_qp_attr.cap.max_recv_sge = 1;
+ init_qp_attr.qp_context = node;
+ init_qp_attr.sq_sig_all = 1;
+ init_qp_attr.qp_type = IBV_QPT_RC;
+ init_qp_attr.send_cq = node->cq;
+ init_qp_attr.recv_cq = node->cq;
+ ret = rdma_create_qp(node->cma_id, node->pd, &init_qp_attr);
+ if (ret) {
+ printf("cmatose: unable to create QP: %d\n", ret);
+ goto out;
+ }
+
+ ret = create_message(node);
+ if (ret) {
+ printf("cmatose: failed to create messages: %d\n", ret);
+ goto out;
+ }
+out:
+ return ret;
+}
+
+static int post_recvs(struct cmatest_node *node)
+{
+ struct ibv_recv_wr recv_wr, *recv_failure;
+ struct ibv_sge sge;
+ int i, ret = 0;
+
+ if (!message_count)
+ return 0;
+
+ recv_wr.next = NULL;
+ recv_wr.sg_list = &sge;
+ recv_wr.num_sge = 1;
+ recv_wr.wr_id = (uintptr_t) node;
+
+ sge.length = message_size;
+ sge.lkey = node->mr->lkey;
+ sge.addr = (uintptr_t) node->mem;
+
+ for (i = 0; i < message_count && !ret; i++ ) {
+ ret = ibv_post_recv(node->cma_id->qp, &recv_wr, &recv_failure);
+ if (ret) {
+ printf("failed to post receives: %d\n", ret);
+ break;
+ }
+ }
+ return ret;
+}
+
+static int post_sends(struct cmatest_node *node)
+{
+ struct ibv_send_wr send_wr, *bad_send_wr;
+ struct ibv_sge sge;
+ int i, ret = 0;
+
+ if (!node->connected || !message_count)
+ return 0;
+
+ send_wr.next = NULL;
+ send_wr.sg_list = &sge;
+ send_wr.num_sge = 1;
+ send_wr.opcode = IBV_WR_SEND;
+ send_wr.send_flags = 0;
+ send_wr.wr_id = (unsigned long)node;
+
+ sge.length = message_size;
+ sge.lkey = node->mr->lkey;
+ sge.addr = (uintptr_t) node->mem;
+
+ for (i = 0; i < message_count && !ret; i++)
+ ret = ibv_post_send(node->cma_id->qp, &send_wr, &bad_send_wr);
+
+ return ret;
+}
+
+static void connect_error(void)
+{
+ test.disconnects_left--;
+ test.connects_left--;
+}
+
+static void addr_handler(struct cmatest_node *node)
+{
+ int ret;
+
+ ret = rdma_resolve_route(node->cma_id, 2000);
+ if (ret) {
+ printf("cmatose: resolve route failed: %d\n", ret);
+ connect_error();
+ }
+}
+
+static void route_handler(struct cmatest_node *node)
+{
+ struct rdma_conn_param conn_param;
+ int ret;
+
+ ret = init_node(node);
+ if (ret)
+ goto err;
+
+ ret = post_recvs(node);
+ if (ret)
+ goto err;
+
+ memset(&conn_param, 0, sizeof conn_param);
+ conn_param.responder_resources = 1;
+ conn_param.initiator_depth = 1;
+ conn_param.retry_count = 5;
+ ret = rdma_connect(node->cma_id, &conn_param);
+ if (ret) {
+ printf("cmatose: failure connecting: %d\n", ret);
+ goto err;
+ }
+ return;
+err:
+ connect_error();
+}
+
+static int connect_handler(struct rdma_cm_id *cma_id)
+{
+ struct cmatest_node *node;
+ struct rdma_conn_param conn_param;
+ int ret;
+
+ if (test.conn_index == connections) {
+ ret = -ENOMEM;
+ goto err1;
+ }
+ node = &test.nodes[test.conn_index++];
+
+ node->cma_id = cma_id;
+ cma_id->context = node;
+
+ ret = init_node(node);
+ if (ret)
+ goto err2;
+
+ ret = post_recvs(node);
+ if (ret)
+ goto err2;
+
+ memset(&conn_param, 0, sizeof conn_param);
+ conn_param.responder_resources = 1;
+ conn_param.initiator_depth = 1;
+ ret = rdma_accept(node->cma_id, &conn_param);
+ if (ret) {
+ printf("cmatose: failure accepting: %d\n", ret);
+ goto err2;
+ }
+ return 0;
+
+err2:
+ node->cma_id = NULL;
+ connect_error();
+err1:
+ printf("cmatose: failing connection request\n");
+ rdma_reject(cma_id, NULL, 0);
+ return ret;
+}
+
+static int cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
+{
+ int ret = 0;
+
+ switch (event->event) {
+ case RDMA_CM_EVENT_ADDR_RESOLVED:
+ addr_handler(cma_id->context);
+ break;
+ case RDMA_CM_EVENT_ROUTE_RESOLVED:
+ route_handler(cma_id->context);
+ break;
+ case RDMA_CM_EVENT_CONNECT_REQUEST:
+ ret = connect_handler(cma_id);
+ break;
+ case RDMA_CM_EVENT_ESTABLISHED:
+ ((struct cmatest_node *) cma_id->context)->connected = 1;
+ test.connects_left--;
+ break;
+ case RDMA_CM_EVENT_ADDR_ERROR:
+ case RDMA_CM_EVENT_ROUTE_ERROR:
+ case RDMA_CM_EVENT_CONNECT_ERROR:
+ case RDMA_CM_EVENT_UNREACHABLE:
+ case RDMA_CM_EVENT_REJECTED:
+ printf("cmatose: event: %d, error: %d\n", event->event,
+ event->status);
+ connect_error();
+ break;
+ case RDMA_CM_EVENT_DISCONNECTED:
+ rdma_disconnect(cma_id);
+ test.disconnects_left--;
+ break;
+ case RDMA_CM_EVENT_DEVICE_REMOVAL:
+ /* Cleanup will occur after test completes. */
+ break;
+ default:
+ break;
+ }
+ return ret;
+}
+
+static void destroy_node(struct cmatest_node *node)
+{
+ if (!node->cma_id)
+ return;
+
+ if (node->cma_id->qp)
+ rdma_destroy_qp(node->cma_id);
+
+ if (node->cq)
+ ibv_destroy_cq(node->cq);
+
+ if (node->mem) {
+ ibv_dereg_mr(node->mr);
+ free(node->mem);
+ }
+
+ if (node->pd)
+ ibv_dealloc_pd(node->pd);
+
+ /* Destroy the RDMA ID after all device resources */
+ rdma_destroy_id(node->cma_id);
+}
+
+static int alloc_nodes(void)
+{
+ int ret, i;
+
+ test.nodes = malloc(sizeof *test.nodes * connections);
+ if (!test.nodes) {
+ printf("cmatose: unable to allocate memory for test nodes\n");
+ return -ENOMEM;
+ }
+ memset(test.nodes, 0, sizeof *test.nodes * connections);
+
+ for (i = 0; i < connections; i++) {
+ test.nodes[i].id = i;
+ if (!is_server) {
+ ret = rdma_create_id(&test.nodes[i].cma_id,
+ &test.nodes[i]);
+ if (ret)
+ goto err;
+ }
+ }
+ return 0;
+err:
+ while (--i >= 0)
+ rdma_destroy_id(test.nodes[i].cma_id);
+ free(test.nodes);
+ return ret;
+}
+
+static void destroy_nodes(void)
+{
+ int i;
+
+ for (i = 0; i < connections; i++)
+ destroy_node(&test.nodes[i]);
+ free(test.nodes);
+}
+
+static int poll_cqs(void)
+{
+ struct ibv_wc wc[8];
+ int done, i, ret;
+
+ for (i = 0; i < connections; i++) {
+ if (!test.nodes[i].connected)
+ continue;
+
+ for (done = 0; done < message_count; done += ret) {
+ ret = ibv_poll_cq(test.nodes[i].cq, 8, wc);
+ if (ret < 0) {
+ printf("cmatose: failed polling CQ: %d\n", ret);
+ return ret;
+ }
+ }
+ }
+ return 0;
+}
+
+static void connect_events(void)
+{
+ struct rdma_cm_event *event;
+ int err = 0;
+
+ while (test.connects_left && !err) {
+ err = rdma_get_cm_event(&event);
+ if (!err) {
+ cma_handler(event->id, event);
+ rdma_ack_cm_event(event);
+ }
+ }
+}
+
+static void disconnect_events(void)
+{
+ struct rdma_cm_event *event;
+ int err = 0;
+
+ while (test.disconnects_left && !err) {
+ err = rdma_get_cm_event(&event);
+ if (!err) {
+ cma_handler(event->id, event);
+ rdma_ack_cm_event(event);
+ }
+ }
+}
+
+static void run_server(void)
+{
+ struct rdma_cm_id *listen_id;
+ int i, ret;
+
+ printf("cmatose: starting server\n");
+ ret = rdma_create_id(&listen_id, &test);
+ if (ret) {
+ printf("cmatose: listen request failed\n");
+ return;
+ }
+
+ test.addr_in.sin_family = PF_INET;
+ test.addr_in.sin_port = 7471;
+ ret = rdma_bind_addr(listen_id, test.addr);
+ if (ret) {
+ printf("cmatose: bind address failed: %d\n", ret);
+ return;
+ }
+
+ ret = rdma_listen(listen_id, 0);
+ if (ret) {
+ printf("cmatose: failure trying to listen: %d\n", ret);
+ goto out;
+ }
+
+ connect_events();
+
+ if (message_count) {
+ printf("initiating data transfers\n");
+ for (i = 0; i < connections; i++)
+ if (post_sends(&test.nodes[i]))
+ goto out;
+
+ printf("receiving data transfers\n");
+ if (poll_cqs())
+ goto out;
+ printf("data transfers complete\n");
+
+ }
+ printf("cmatose: disconnecting\n");
+ for (i = 0; i < connections; i++) {
+ if (!test.nodes[i].connected)
+ continue;
+
+ test.nodes[i].connected = 0;
+ rdma_disconnect(test.nodes[i].cma_id);
+ }
+
+ disconnect_events();
+ printf("disconnected\n");
+out:
+ rdma_destroy_id(listen_id);
+}
+
+static int get_dst_addr(char *dst)
+{
+ struct addrinfo *res;
+ int ret;
+
+ ret = getaddrinfo(dst, NULL, NULL, &res);
+ if (ret) {
+ printf("getaddrinfo failed - invalid hostname or IP address\n");
+ return ret;
+ }
+
+ if (res->ai_family != PF_INET) {
+ ret = -1;
+ goto out;
+ }
+
+ test.addr_in = *(struct sockaddr_in *) res->ai_addr;
+ test.addr_in.sin_port = 7471;
+out:
+ freeaddrinfo(res);
+ return ret;
+}
+
+static void run_client(char *dst)
+{
+ int i, ret;
+
+ printf("cmatose: starting client\n");
+ ret = get_dst_addr(dst);
+ if (ret)
+ return;
+
+ printf("cmatose: connecting\n");
+ for (i = 0; i < connections; i++) {
+ ret = rdma_resolve_addr(test.nodes[i].cma_id, NULL,
+ test.addr, 2000);
+ if (ret) {
+ printf("cmatose: failure getting addr: %d\n", ret);
+ connect_error();
+ }
+ }
+
+ connect_events();
+
+ if (message_count) {
+ printf("receiving data transfers\n");
+ if (poll_cqs())
+ goto out;
+
+ printf("sending replies\n");
+ for (i = 0; i < connections; i++)
+ if (post_sends(&test.nodes[i]))
+ goto out;
+
+ printf("data transfers complete\n");
+
+ }
+out:
+ disconnect_events();
+}
+
+int main(int argc, char **argv)
+{
+ if (argc != 1 && argc != 2) {
+ printf("usage: %s [server_addr]\n", argv[0]);
+ exit(1);
+ }
+ is_server = (argc == 1);
+
+ test.addr = (struct sockaddr *) &test.addr_in;
+ test.connects_left = connections;
+ test.disconnects_left = connections;
+ if (alloc_nodes())
+ exit(1);
+
+ if (is_server)
+ run_server();
+ else
+ run_client(argv[1]);
+
+ printf("test complete\n");
+ destroy_nodes();
+ return 0;
+}
--- /dev/null
+/*
+ * Copyright (c) 2005 Voltaire Inc. All rights reserved.
+ * Copyright (c) 2005 Intel Corporation. All rights reserved.
+ *
+ * This Software is licensed under one of the following licenses:
+ *
+ * 1) under the terms of the "Common Public License 1.0" a copy of which is
+ * available from the Open Source Initiative, see
+ * http://www.opensource.org/licenses/cpl.php.
+ *
+ * 2) under the terms of the "The BSD License" a copy of which is
+ * available from the Open Source Initiative, see
+ * http://www.opensource.org/licenses/bsd-license.php.
+ *
+ * 3) under the terms of the "GNU General Public License (GPL) Version 2" a
+ * copy of which is available from the Open Source Initiative, see
+ * http://www.opensource.org/licenses/gpl-license.php.
+ *
+ * Licensee has the right to choose one of the above licenses.
+ *
+ * Redistributions of source code must retain the above copyright
+ * notice and one of the license notices.
+ *
+ * Redistributions in binary form must reproduce both the above copyright
+ * notice, one of the license notices in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ */
+
+#if !defined(RDMA_CMA_H)
+#define RDMA_CMA_H
+
+#include <netinet/in.h>
+#include <sys/socket.h>
+#include <infiniband/verbs.h>
+#include <infiniband/sa.h>
+
+/*
+ * Upon receiving a device removal event, users must destroy the associated
+ * RDMA identifier and release all resources allocated with the device.
+ */
+enum rdma_cm_event_type {
+ RDMA_CM_EVENT_ADDR_RESOLVED,
+ RDMA_CM_EVENT_ADDR_ERROR,
+ RDMA_CM_EVENT_ROUTE_RESOLVED,
+ RDMA_CM_EVENT_ROUTE_ERROR,
+ RDMA_CM_EVENT_CONNECT_REQUEST,
+ RDMA_CM_EVENT_CONNECT_RESPONSE,
+ RDMA_CM_EVENT_CONNECT_ERROR,
+ RDMA_CM_EVENT_UNREACHABLE,
+ RDMA_CM_EVENT_REJECTED,
+ RDMA_CM_EVENT_ESTABLISHED,
+ RDMA_CM_EVENT_DISCONNECTED,
+ RDMA_CM_EVENT_DEVICE_REMOVAL,
+};
+
+struct ib_addr {
+ union ibv_gid sgid;
+ union ibv_gid dgid;
+ uint16_t pkey;
+};
+
+struct rdma_addr {
+ struct sockaddr_in6 src_addr;
+ struct sockaddr_in6 dst_addr;
+ union {
+ struct ib_addr ibaddr;
+ } addr;
+};
+
+struct rdma_route {
+ struct rdma_addr addr;
+ struct ib_sa_path_rec *path_rec;
+ int num_paths;
+};
+
+struct rdma_cm_id {
+ struct ibv_context *verbs;
+ void *context;
+ struct ibv_qp *qp;
+ struct rdma_route route;
+};
+
+struct rdma_cm_event {
+ struct rdma_cm_id *id;
+ struct rdma_cm_id *listen_id;
+ enum rdma_cm_event_type event;
+ int status;
+ void *private_data;
+ uint8_t private_data_len;
+};
+
+int rdma_create_id(struct rdma_cm_id **id, void *context);
+
+int rdma_destroy_id(struct rdma_cm_id *id);
+
+/**
+ * rdma_bind_addr - Bind an RDMA identifier to a source address and
+ * associated RDMA device, if needed.
+ *
+ * @id: RDMA identifier.
+ * @addr: Local address information. Wildcard values are permitted.
+ *
+ * This associates a source address with the RDMA identifier before calling
+ * rdma_listen. If a specific local address is given, the RDMA identifier will
+ * be bound to a local RDMA device.
+ */
+int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr);
+
+/**
+ * rdma_resolve_addr - Resolve destination and optional source addresses
+ * from IP addresses to an RDMA address. If successful, the specified
+ * rdma_cm_id will be bound to a local device.
+ *
+ * @id: RDMA identifier.
+ * @src_addr: Source address information. This parameter may be NULL.
+ * @dst_addr: Destination address information.
+ * @timeout_ms: Time to wait for resolution to complete.
+ */
+int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
+ struct sockaddr *dst_addr, int timeout_ms);
+
+/**
+ * rdma_resolve_route - Resolve the RDMA address bound to the RDMA identifier
+ * into route information needed to establish a connection.
+ *
+ * This is called on the client side of a connection.
+ * Users must have first called rdma_resolve_addr to resolve a dst_addr
+ * into an RDMA address before calling this routine.
+ */
+int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms);
+
+/**
+ * rdma_create_qp - Allocate a QP and associate it with the specified RDMA
+ * identifier.
+ *
+ * QPs allocated to an rdma_cm_id will automatically be transitioned by the CMA
+ * through their states.
+ */
+int rdma_create_qp(struct rdma_cm_id *id, struct ibv_pd *pd,
+ struct ibv_qp_init_attr *qp_init_attr);
+
+/**
+ * rdma_destroy_qp - Deallocate the QP associated with the specified RDMA
+ * identifier.
+ *
+ * Users must destroy any QP associated with an RDMA identifier before
+ * destroying the RDMA ID.
+ */
+void rdma_destroy_qp(struct rdma_cm_id *id);
+
+struct rdma_conn_param {
+ const void *private_data;
+ uint8_t private_data_len;
+ uint8_t responder_resources;
+ uint8_t initiator_depth;
+ uint8_t flow_control;
+ uint8_t retry_count; /* ignored when accepting */
+ uint8_t rnr_retry_count;
+};
+
+/**
+ * rdma_connect - Initiate an active connection request.
+ *
+ * Users must have resolved a route for the rdma_cm_id to connect with
+ * by having called rdma_resolve_route before calling this routine.
+ */
+int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param);
+
+/**
+ * rdma_listen - This function is called by the passive side to
+ * listen for incoming connection requests.
+ *
+ * Users must have bound the rdma_cm_id to a local address by calling
+ * rdma_bind_addr before calling this routine.
+ */
+int rdma_listen(struct rdma_cm_id *id, int backlog);
+
+/**
+ * rdma_accept - Called to accept a connection request.
+ * @id: Connection identifier associated with the request.
+ * @conn_param: Information needed to establish the connection.
+ */
+int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param);
+
+/**
+ * rdma_reject - Called on the passive side to reject a connection request.
+ */
+int rdma_reject(struct rdma_cm_id *id, const void *private_data,
+ uint8_t private_data_len);
+
+/**
+ * rdma_disconnect - This function disconnects the associated QP.
+ */
+int rdma_disconnect(struct rdma_cm_id *id);
+
+/**
+ * rdma_get_cm_event - Retrieves the next pending communications event,
+ * if no event is pending waits for an event.
+ * @event: Allocated information about the next communication event.
+ * Event should be freed using rdma_ack_cm_event()
+ *
+ * A RDMA_CM_EVENT_CONNECT_REQUEST communication events result
+ * in the allocation of a new @rdma_cm_id.
+ * Clients are responsible for destroying the new @rdma_cm_id.
+ */
+int rdma_get_cm_event(struct rdma_cm_event **event);
+
+/**
+ * rdma_ack_cm_event - Free a communications event.
+ * @event: Event to be released.
+ *
+ * All events which are allocated by rdma_get_cm_event() must be released,
+ * there should be a one-to-one correspondence between successful gets
+ * and acks.
+ */
+int rdma_ack_cm_event(struct rdma_cm_event *event);
+
+#endif /* RDMA_CMA_H */
--- /dev/null
+/*
+ * Copyright (c) 2005 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef RDMA_CMA_ABI_H
+#define RDMA_CMA_ABI_H
+
+#include <infiniband/sa-kern-abi.h>
+
+/*
+ * This file must be kept in sync with the kernel's version of rdma_user_cm.h
+ */
+
+#define RDMA_USER_CM_MIN_ABI_VERSION 1
+#define RDMA_USER_CM_MAX_ABI_VERSION 1
+
+#define RDMA_MAX_PRIVATE_DATA 256
+
+enum {
+ UCMA_CMD_CREATE_ID,
+ UCMA_CMD_DESTROY_ID,
+ UCMA_CMD_BIND_ADDR,
+ UCMA_CMD_RESOLVE_ADDR,
+ UCMA_CMD_RESOLVE_ROUTE,
+ UCMA_CMD_QUERY_ROUTE,
+ UCMA_CMD_CONNECT,
+ UCMA_CMD_LISTEN,
+ UCMA_CMD_ACCEPT,
+ UCMA_CMD_REJECT,
+ UCMA_CMD_DISCONNECT,
+ UCMA_CMD_INIT_QP_ATTR,
+ UCMA_CMD_GET_EVENT
+};
+
+struct ucma_abi_cmd_hdr {
+ __u32 cmd;
+ __u16 in;
+ __u16 out;
+};
+
+struct ucma_abi_create_id {
+ __u64 uid;
+ __u64 response;
+};
+
+struct ucma_abi_create_id_resp {
+ __u32 id;
+};
+
+struct ucma_abi_destroy_id {
+ __u64 response;
+ __u32 id;
+ __u32 reserved;
+};
+
+struct ucma_abi_destroy_id_resp {
+ __u32 events_reported;
+};
+
+struct ucma_abi_bind_addr {
+ __u64 response;
+ struct sockaddr_in6 addr;
+ __u32 id;
+};
+
+struct ucma_abi_bind_addr_resp {
+ __u64 node_guid;
+};
+
+struct ucma_abi_resolve_addr {
+ struct sockaddr_in6 src_addr;
+ struct sockaddr_in6 dst_addr;
+ __u32 id;
+ __u32 timeout_ms;
+};
+
+struct ucma_abi_resolve_route {
+ __u32 id;
+ __u32 timeout_ms;
+};
+
+struct ucma_abi_query_route {
+ __u64 response;
+ __u32 id;
+ __u32 reserved;
+};
+
+struct ucma_abi_query_route_resp {
+ __u64 node_guid;
+ struct ib_kern_path_rec ib_route[2];
+ struct sockaddr_in6 src_addr;
+ __u32 num_paths;
+};
+
+struct ucma_abi_conn_param {
+ __u32 qp_num;
+ __u32 qp_type;
+ __u8 private_data[RDMA_MAX_PRIVATE_DATA];
+ __u8 private_data_len;
+ __u8 srq;
+ __u8 responder_resources;
+ __u8 initiator_depth;
+ __u8 flow_control;
+ __u8 retry_count;
+ __u8 rnr_retry_count;
+ __u8 valid;
+};
+
+struct ucma_abi_connect {
+ struct ucma_abi_conn_param conn_param;
+ __u32 id;
+ __u32 reserved;
+};
+
+struct ucma_abi_listen {
+ __u32 id;
+ __u32 backlog;
+};
+
+struct ucma_abi_accept {
+ __u64 uid;
+ struct ucma_abi_conn_param conn_param;
+ __u32 id;
+ __u32 reserved;
+};
+
+struct ucma_abi_reject {
+ __u32 id;
+ __u8 private_data_len;
+ __u8 reserved[3];
+ __u8 private_data[RDMA_MAX_PRIVATE_DATA];
+};
+
+struct ucma_abi_disconnect {
+ __u32 id;
+};
+
+struct ucma_abi_init_qp_attr {
+ __u64 response;
+ __u32 id;
+ __u32 qp_state;
+};
+
+struct ucma_abi_get_event {
+ __u64 response;
+};
+
+struct ucma_abi_event_resp {
+ __u64 uid;
+ __u32 id;
+ __u32 event;
+ __u32 status;
+ __u8 private_data_len;
+ __u8 reserved[3];
+ __u8 private_data[RDMA_MAX_PRIVATE_DATA];
+};
+
+#endif /* RDMA_CMA_ABI_H */
--- /dev/null
+# $Id: $
+
+%define prefix /usr
+%define ver @VERSION@
+%define RELEASE 1
+%define rel %{?CUSTOM_RELEASE} %{!?CUSTOM_RELEASE:%RELEASE}
+
+Summary: Userspace RDMA Connection Manager.
+Name: librdmacm
+Version: %ver
+Release: %rel
+Copyright: Dual GPL/BSD
+Group: System Environment/Libraries
+BuildRoot: %{_tmppath}/%{name}-%{version}-root
+Source: http://openib.org/downloads/%{name}-%{version}.tar.gz
+Url: http://openib.org/
+
+%description
+Along with the OpenIB kernel drivers, librdmacm provides a userspace
+RDMA Connection Managment API.
+
+%prep
+%setup -q
+
+%build
+%configure
+make
+
+%install
+make DESTDIR=${RPM_BUILD_ROOT} install
+# remove unpackaged files from the buildroot
+rm -f $RPM_BUILD_ROOT%{_libdir}/*.la
+
+%clean
+rm -rf $RPM_BUILD_ROOT
+
+%files
+%defattr(-,root,root)
+%{_libdir}/librdmacm*.so.*
+%doc AUTHORS COPYING ChangeLog NEWS README
--- /dev/null
+/*
+ * Copyright (c) 2005 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id: cm.c 3453 2005-09-15 21:43:21Z sean.hefty $
+ */
+
+#if HAVE_CONFIG_H
+# include <config.h>
+#endif /* HAVE_CONFIG_H */
+
+#include <stdlib.h>
+#include <string.h>
+#include <glob.h>
+#include <stdio.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <stdint.h>
+#include <poll.h>
+#include <unistd.h>
+#include <pthread.h>
+#include <endian.h>
+#include <byteswap.h>
+
+#include <infiniband/marshall.h>
+#include <rdma/rdma_cma.h>
+#include <rdma/rdma_cma_abi.h>
+
+#define PFX "librdmacm: "
+
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+static inline uint64_t htonll(uint64_t x) { return bswap_64(x); }
+static inline uint64_t ntohll(uint64_t x) { return bswap_64(x); }
+#else
+static inline uint64_t htonll(uint64_t x) { return x; }
+static inline uint64_t ntohll(uint64_t x) { return x; }
+#endif
+
+#define CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, type, size) \
+do { \
+ struct ucma_abi_cmd_hdr *hdr; \
+ \
+ size = sizeof(*hdr) + sizeof(*cmd); \
+ msg = alloca(size); \
+ if (!msg) \
+ return -ENOMEM; \
+ hdr = msg; \
+ cmd = msg + sizeof(*hdr); \
+ hdr->cmd = type; \
+ hdr->in = sizeof(*cmd); \
+ hdr->out = sizeof(*resp); \
+ memset(cmd, 0, sizeof(*cmd)); \
+ resp = alloca(sizeof(*resp)); \
+ if (!resp) \
+ return -ENOMEM; \
+ cmd->response = (uintptr_t)resp;\
+} while (0)
+
+#define CMA_CREATE_MSG_CMD(msg, cmd, type, size) \
+do { \
+ struct ucma_abi_cmd_hdr *hdr; \
+ \
+ size = sizeof(*hdr) + sizeof(*cmd); \
+ msg = alloca(size); \
+ if (!msg) \
+ return -ENOMEM; \
+ hdr = msg; \
+ cmd = msg + sizeof(*hdr); \
+ hdr->cmd = type; \
+ hdr->in = sizeof(*cmd); \
+ hdr->out = 0; \
+ memset(cmd, 0, sizeof(*cmd)); \
+} while (0)
+
+struct cma_device {
+ struct ibv_context *verbs;
+ uint64_t guid;
+ int port_cnt;
+};
+
+struct cma_id_private {
+ struct rdma_cm_id id;
+ struct cma_device *cma_dev;
+ int events_completed;
+ pthread_cond_t cond;
+ pthread_mutex_t mut;
+ uint32_t handle;
+};
+
+static struct dlist *dev_list;
+static struct dlist *cma_dev_list;
+int cma_fd;
+
+#define container_of(ptr, type, field) \
+ ((type *) ((void *)ptr - offsetof(type, field)))
+
+static void __attribute__((constructor)) rdma_cma_init(void)
+{
+ struct ibv_device *dev;
+ struct cma_device *cma_dev;
+ struct ibv_device_attr attr;
+ int ret;
+
+ cma_fd = open("/dev/infiniband/rdma_cm", O_RDWR);
+ if (cma_fd < 0)
+ abort();
+
+ cma_dev_list = dlist_new(sizeof *cma_dev);
+ dev_list = ibv_get_devices();
+ if (!cma_dev_list || !dev_list)
+ abort();
+
+ dlist_for_each_data(dev_list, dev, struct ibv_device) {
+ cma_dev = malloc(sizeof *cma_dev);
+ if (!cma_dev)
+ abort();
+
+ cma_dev->guid = ibv_get_device_guid(dev);
+ cma_dev->verbs = ibv_open_device(dev);
+ if (!cma_dev->verbs)
+ abort();
+
+ ret = ibv_query_device(cma_dev->verbs, &attr);
+ if (ret)
+ abort();
+
+ cma_dev->port_cnt = attr.phys_port_cnt;
+ dlist_push(cma_dev_list, cma_dev);
+ }
+}
+
+static void __attribute__((destructor)) rdma_cma_fini(void)
+{
+ struct cma_device *cma_dev;
+
+ if (!cma_dev_list)
+ return;
+
+ dlist_for_each_data(cma_dev_list, cma_dev, struct cma_device)
+ ibv_close_device(cma_dev->verbs);
+
+ dlist_destroy(cma_dev_list);
+ close(cma_fd);
+}
+
+static int ucma_get_device(struct cma_id_private *id_priv, uint64_t guid)
+{
+ struct cma_device *cma_dev;
+
+ dlist_for_each_data(cma_dev_list, cma_dev, struct cma_device)
+ if (cma_dev->guid == guid) {
+ id_priv->cma_dev = cma_dev;
+ id_priv->id.verbs = cma_dev->verbs;
+ return 0;
+ }
+
+ return -ENODEV;
+}
+
+static void ucma_free_id(struct cma_id_private *id_priv)
+{
+ pthread_cond_destroy(&id_priv->cond);
+ pthread_mutex_destroy(&id_priv->mut);
+ if (id_priv->id.route.path_rec)
+ free(id_priv->id.route.path_rec);
+ free(id_priv);
+}
+
+static struct cma_id_private *ucma_alloc_id(void *context)
+{
+ struct cma_id_private *id_priv;
+
+ id_priv = malloc(sizeof *id_priv);
+ if (!id_priv)
+ return NULL;
+
+ memset(id_priv, 0, sizeof *id_priv);
+ id_priv->id.context = context;
+ pthread_mutex_init(&id_priv->mut, NULL);
+ if (pthread_cond_init(&id_priv->cond, NULL))
+ goto err;
+
+ return id_priv;
+
+err: ucma_free_id(id_priv);
+ return NULL;
+}
+
+int rdma_create_id(struct rdma_cm_id **id, void *context)
+{
+ struct ucma_abi_create_id_resp *resp;
+ struct ucma_abi_create_id *cmd;
+ struct cma_id_private *id_priv;
+ void *msg;
+ int ret, size;
+
+ id_priv = ucma_alloc_id(context);
+ if (!id_priv)
+ return -ENOMEM;
+
+ CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_CREATE_ID, size);
+ cmd->uid = (uintptr_t) id_priv;
+
+ ret = write(cma_fd, msg, size);
+ if (ret != size)
+ goto err;
+
+ id_priv->handle = resp->id;
+ *id = &id_priv->id;
+ return 0;
+
+err: ucma_free_id(id_priv);
+ return ret;
+}
+
+static int ucma_destroy_kern_id(uint32_t handle)
+{
+ struct ucma_abi_destroy_id_resp *resp;
+ struct ucma_abi_destroy_id *cmd;
+ void *msg;
+ int ret, size;
+
+ CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_DESTROY_ID, size);
+ cmd->id = handle;
+
+ ret = write(cma_fd, msg, size);
+ if (ret != size)
+ return (ret > 0) ? -ENODATA : ret;
+
+ return resp->events_reported;
+}
+
+int rdma_destroy_id(struct rdma_cm_id *id)
+{
+ struct cma_id_private *id_priv;
+ int ret;
+
+ id_priv = container_of(id, struct cma_id_private, id);
+ ret = ucma_destroy_kern_id(id_priv->handle);
+ if (ret < 0)
+ return ret;
+
+ pthread_mutex_lock(&id_priv->mut);
+ while (id_priv->events_completed < ret)
+ pthread_cond_wait(&id_priv->cond, &id_priv->mut);
+ pthread_mutex_unlock(&id_priv->mut);
+
+ ucma_free_id(id_priv);
+ return 0;
+}
+
+static int ucma_addrlen(struct sockaddr *addr)
+{
+ if (!addr)
+ return 0;
+
+ switch (addr->sa_family) {
+ case PF_INET:
+ return sizeof(struct sockaddr_in);
+ case PF_INET6:
+ return sizeof(struct sockaddr_in6);
+ default:
+ return 0;
+ }
+}
+
+int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
+{
+ struct ucma_abi_bind_addr_resp *resp;
+ struct ucma_abi_bind_addr *cmd;
+ struct cma_id_private *id_priv;
+ void *msg;
+ int ret, size, addrlen;
+
+ addrlen = ucma_addrlen(addr);
+ if (!addrlen)
+ return -EINVAL;
+
+ CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_BIND_ADDR, size);
+ id_priv = container_of(id, struct cma_id_private, id);
+ cmd->id = id_priv->handle;
+ memcpy(&cmd->addr, addr, addrlen);
+
+ ret = write(cma_fd, msg, size);
+ if (ret != size)
+ return (ret > 0) ? -ENODATA : ret;
+
+ if (resp->node_guid) {
+ ret = ucma_get_device(id_priv, resp->node_guid);
+ if (ret)
+ return ret;
+ }
+
+ memcpy(&id->route.addr.src_addr, addr, addrlen);
+ return 0;
+}
+
+int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
+ struct sockaddr *dst_addr, int timeout_ms)
+{
+ struct ucma_abi_resolve_addr *cmd;
+ struct cma_id_private *id_priv;
+ void *msg;
+ int ret, size, daddrlen;
+
+ daddrlen = ucma_addrlen(dst_addr);
+ if (!daddrlen)
+ return -EINVAL;
+
+ CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_RESOLVE_ADDR, size);
+ id_priv = container_of(id, struct cma_id_private, id);
+ cmd->id = id_priv->handle;
+ if (src_addr)
+ memcpy(&cmd->src_addr, src_addr, ucma_addrlen(src_addr));
+ memcpy(&cmd->dst_addr, dst_addr, daddrlen);
+ cmd->timeout_ms = timeout_ms;
+
+ ret = write(cma_fd, msg, size);
+ if (ret != size)
+ return (ret > 0) ? -ENODATA : ret;
+
+ memcpy(&id->route.addr.dst_addr, dst_addr, daddrlen);
+ return 0;
+}
+
+int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)
+{
+ struct ucma_abi_resolve_route *cmd;
+ struct cma_id_private *id_priv;
+ void *msg;
+ int ret, size;
+
+ CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_RESOLVE_ROUTE, size);
+ id_priv = container_of(id, struct cma_id_private, id);
+ cmd->id = id_priv->handle;
+ cmd->timeout_ms = timeout_ms;
+
+ ret = write(cma_fd, msg, size);
+ if (ret != size)
+ return (ret > 0) ? -ENODATA : ret;
+
+ return 0;
+}
+
+static int rdma_init_qp_attr(struct rdma_cm_id *id, struct ibv_qp_attr *qp_attr,
+ int *qp_attr_mask)
+{
+ struct ucma_abi_init_qp_attr *cmd;
+ struct ibv_kern_qp_attr *resp;
+ struct cma_id_private *id_priv;
+ void *msg;
+ int ret, size;
+
+ CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_INIT_QP_ATTR, size);
+ id_priv = container_of(id, struct cma_id_private, id);
+ cmd->id = id_priv->handle;
+ cmd->qp_state = qp_attr->qp_state;
+
+ ret = write(cma_fd, msg, size);
+ if (ret != size)
+ return (ret > 0) ? -ENODATA : ret;
+
+ ib_copy_qp_attr_from_kern(qp_attr, resp);
+ *qp_attr_mask = resp->qp_attr_mask;
+ return 0;
+}
+
+static int ucma_modify_qp_rtr(struct rdma_cm_id *id)
+{
+ struct ibv_qp_attr qp_attr;
+ int qp_attr_mask, ret;
+
+ if (!id->qp)
+ return -EINVAL;
+
+ /* Need to update QP attributes from default values. */
+ qp_attr.qp_state = IBV_QPS_INIT;
+ ret = rdma_init_qp_attr(id, &qp_attr, &qp_attr_mask);
+ if (ret)
+ return ret;
+
+ ret = ibv_modify_qp(id->qp, &qp_attr, qp_attr_mask);
+ if (ret)
+ return ret;
+
+ qp_attr.qp_state = IBV_QPS_RTR;
+ ret = rdma_init_qp_attr(id, &qp_attr, &qp_attr_mask);
+ if (ret)
+ return ret;
+
+ return ibv_modify_qp(id->qp, &qp_attr, qp_attr_mask);
+}
+
+static int ucma_modify_qp_rts(struct rdma_cm_id *id)
+{
+ struct ibv_qp_attr qp_attr;
+ int qp_attr_mask, ret;
+
+ qp_attr.qp_state = IBV_QPS_RTS;
+ ret = rdma_init_qp_attr(id, &qp_attr, &qp_attr_mask);
+ if (ret)
+ return ret;
+
+ return ibv_modify_qp(id->qp, &qp_attr, qp_attr_mask);
+}
+
+static int ucma_modify_qp_err(struct rdma_cm_id *id)
+{
+ struct ibv_qp_attr qp_attr;
+
+ if (!id->qp)
+ return 0;
+
+ qp_attr.qp_state = IBV_QPS_ERR;
+ return ibv_modify_qp(id->qp, &qp_attr, IBV_QP_STATE);
+}
+
+static int ucma_find_gid(struct cma_device *cma_dev, union ibv_gid *gid,
+ uint8_t *port_num)
+{
+ int port, ret, i;
+ union ibv_gid chk_gid;
+
+ for (port = 1; port <= cma_dev->port_cnt; port++)
+ for (i = 0, ret = 0; !ret; i++) {
+ ret = ibv_query_gid(cma_dev->verbs, port, i, &chk_gid);
+ if (!ret && !memcmp(gid, &chk_gid, sizeof *gid)) {
+ *port_num = port;
+ return 0;
+ }
+ }
+
+ return -EINVAL;
+}
+
+static int ucma_find_pkey(struct cma_device *cma_dev, uint8_t port_num,
+ uint16_t pkey, uint16_t *pkey_index)
+{
+ int ret, i;
+ uint16_t chk_pkey;
+
+ for (i = 0, ret = 0; !ret; i++) {
+ ret = ibv_query_pkey(cma_dev->verbs, port_num, i, &chk_pkey);
+ if (!ret && pkey == chk_pkey) {
+ *pkey_index = (uint16_t) i;
+ return 0;
+ }
+ }
+
+ return -EINVAL;
+}
+
+static int ucma_init_ib_qp(struct cma_id_private *id_priv, struct ibv_qp *qp)
+{
+ struct ibv_qp_attr qp_attr;
+ struct ib_addr *ibaddr;
+ int ret;
+
+ qp_attr.qp_state = IBV_QPS_INIT;
+ qp_attr.qp_access_flags = IBV_ACCESS_LOCAL_WRITE;
+
+ ibaddr = &id_priv->id.route.addr.addr.ibaddr;
+ ret = ucma_find_gid(id_priv->cma_dev, &ibaddr->sgid, &qp_attr.port_num);
+ if (ret)
+ return ret;
+
+ ret = ucma_find_pkey(id_priv->cma_dev, qp_attr.port_num, ibaddr->pkey,
+ &qp_attr.pkey_index);
+ if (ret)
+ return ret;
+
+ return ibv_modify_qp(qp, &qp_attr, IBV_QP_STATE | IBV_QP_ACCESS_FLAGS |
+ IBV_QP_PKEY_INDEX | IBV_QP_PORT);
+}
+
+int rdma_create_qp(struct rdma_cm_id *id, struct ibv_pd *pd,
+ struct ibv_qp_init_attr *qp_init_attr)
+{
+ struct cma_id_private *id_priv;
+ struct ibv_qp *qp;
+ int ret;
+
+ id_priv = container_of(id, struct cma_id_private, id);
+ if (id->verbs != pd->context)
+ return -EINVAL;
+
+ qp = ibv_create_qp(pd, qp_init_attr);
+ if (!qp)
+ return -ENOMEM;
+
+ ret = ucma_init_ib_qp(id_priv, qp);
+ if (ret)
+ goto err;
+
+ id->qp = qp;
+ return 0;
+err:
+ ibv_destroy_qp(qp);
+ return ret;
+}
+
+void rdma_destroy_qp(struct rdma_cm_id *id)
+{
+ ibv_destroy_qp(id->qp);
+}
+
+static int ucma_query_route(struct rdma_cm_id *id)
+{
+ struct ucma_abi_query_route_resp *resp;
+ struct ucma_abi_query_route *cmd;
+ struct cma_id_private *id_priv;
+ void *msg;
+ int ret, size, i;
+
+ CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_QUERY_ROUTE, size);
+ id_priv = container_of(id, struct cma_id_private, id);
+ cmd->id = id_priv->handle;
+
+ ret = write(cma_fd, msg, size);
+ if (ret != size)
+ return (ret > 0) ? -ENODATA : ret;
+
+ if (resp->num_paths) {
+ id->route.path_rec = malloc(sizeof *id->route.path_rec *
+ resp->num_paths);
+ if (!id->route.path_rec)
+ return -ENOMEM;
+
+ id->route.num_paths = resp->num_paths;
+ for (i = 0; i < resp->num_paths; i++)
+ ib_copy_path_rec_from_kern(&id->route.path_rec[i],
+ &resp->ib_route[i]);
+ }
+
+ memcpy(id->route.addr.addr.ibaddr.sgid.raw, resp->ib_route[0].sgid,
+ sizeof id->route.addr.addr.ibaddr.sgid);
+ memcpy(id->route.addr.addr.ibaddr.dgid.raw, resp->ib_route[0].dgid,
+ sizeof id->route.addr.addr.ibaddr.dgid);
+ id->route.addr.addr.ibaddr.pkey = resp->ib_route[0].pkey;
+ memcpy(&id->route.addr.src_addr, &resp->src_addr,
+ sizeof id->route.addr.src_addr);
+
+ if (!id_priv->cma_dev) {
+ ret = ucma_get_device(id_priv, resp->node_guid);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+
+static void ucma_copy_conn_param_to_kern(struct ucma_abi_conn_param *dst,
+ struct rdma_conn_param *src,
+ struct ibv_qp *qp)
+{
+ dst->qp_num = qp->qp_num;
+ dst->qp_type = qp->qp_type;
+ dst->srq = (qp->srq != NULL);
+ dst->responder_resources = src->responder_resources;
+ dst->initiator_depth = src->initiator_depth;
+ dst->flow_control = src->flow_control;
+ dst->retry_count = src->retry_count;
+ dst->rnr_retry_count = src->rnr_retry_count;
+ dst->valid = 1;
+
+ if (src->private_data && src->private_data_len) {
+ memcpy(dst->private_data, src->private_data,
+ src->private_data_len);
+ dst->private_data_len = src->private_data_len;
+ } else
+ src->private_data_len = 0;
+}
+
+int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
+{
+ struct ucma_abi_connect *cmd;
+ struct cma_id_private *id_priv;
+ void *msg;
+ int ret, size;
+
+ CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_CONNECT, size);
+ id_priv = container_of(id, struct cma_id_private, id);
+ cmd->id = id_priv->handle;
+ ucma_copy_conn_param_to_kern(&cmd->conn_param, conn_param, id->qp);
+
+ ret = write(cma_fd, msg, size);
+ if (ret != size)
+ return (ret > 0) ? -ENODATA : ret;
+
+ return 0;
+}
+
+int rdma_listen(struct rdma_cm_id *id, int backlog)
+{
+ struct ucma_abi_listen *cmd;
+ struct cma_id_private *id_priv;
+ void *msg;
+ int ret, size;
+
+ CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_LISTEN, size);
+ id_priv = container_of(id, struct cma_id_private, id);
+ cmd->id = id_priv->handle;
+ cmd->backlog = backlog;
+
+ ret = write(cma_fd, msg, size);
+ if (ret != size)
+ return (ret > 0) ? -ENODATA : ret;
+
+ return 0;
+}
+
+int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
+{
+ struct ucma_abi_accept *cmd;
+ struct cma_id_private *id_priv;
+ void *msg;
+ int ret, size;
+
+ ret = ucma_modify_qp_rtr(id);
+ if (ret)
+ return ret;
+
+ CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_ACCEPT, size);
+ id_priv = container_of(id, struct cma_id_private, id);
+ cmd->id = id_priv->handle;
+ cmd->uid = (uintptr_t) id_priv;
+ ucma_copy_conn_param_to_kern(&cmd->conn_param, conn_param, id->qp);
+
+ ret = write(cma_fd, msg, size);
+ if (ret != size) {
+ ucma_modify_qp_err(id);
+ return (ret > 0) ? -ENODATA : ret;
+ }
+
+ return 0;
+}
+
+int rdma_reject(struct rdma_cm_id *id, const void *private_data,
+ uint8_t private_data_len)
+{
+ struct ucma_abi_reject *cmd;
+ struct cma_id_private *id_priv;
+ void *msg;
+ int ret, size;
+
+ CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_REJECT, size);
+
+ id_priv = container_of(id, struct cma_id_private, id);
+ cmd->id = id_priv->handle;
+ if (private_data && private_data_len) {
+ memcpy(cmd->private_data, private_data, private_data_len);
+ cmd->private_data_len = private_data_len;
+ } else
+ cmd->private_data_len = 0;
+
+ ret = write(cma_fd, msg, size);
+ if (ret != size)
+ return (ret > 0) ? -ENODATA : ret;
+
+ return 0;
+}
+
+int rdma_disconnect(struct rdma_cm_id *id)
+{
+ struct ucma_abi_disconnect *cmd;
+ struct cma_id_private *id_priv;
+ void *msg;
+ int ret, size;
+
+ ret = ucma_modify_qp_err(id);
+ if (ret)
+ return ret;
+
+ CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_DISCONNECT, size);
+ id_priv = container_of(id, struct cma_id_private, id);
+ cmd->id = id_priv->handle;
+
+ ret = write(cma_fd, msg, size);
+ if (ret != size)
+ return (ret > 0) ? -ENODATA : ret;
+
+ return 0;
+}
+
+static void ucma_copy_event_from_kern(struct rdma_cm_event *dst,
+ struct ucma_abi_event_resp *src)
+{
+ dst->event = src->event;
+ dst->status = src->status;
+ dst->private_data_len = src->private_data_len;
+ if (src->private_data_len) {
+ dst->private_data = dst + 1;
+ memcpy(dst->private_data, src->private_data,
+ src->private_data_len);
+ } else
+ dst->private_data = NULL;
+}
+
+static void ucma_complete_event(struct cma_id_private *id_priv)
+{
+ pthread_mutex_lock(&id_priv->mut);
+ id_priv->events_completed++;
+ pthread_cond_signal(&id_priv->cond);
+ pthread_mutex_unlock(&id_priv->mut);
+}
+
+int rdma_ack_cm_event(struct rdma_cm_event *event)
+{
+ struct rdma_cm_id *id;
+
+ if (!event)
+ return -EINVAL;
+
+ id = (event->event == RDMA_CM_EVENT_CONNECT_REQUEST) ?
+ event->listen_id : event->id;
+
+ ucma_complete_event(container_of(id, struct cma_id_private, id));
+ free(event);
+ return 0;
+}
+
+static int ucma_process_conn_req(struct rdma_cm_event *event,
+ uint32_t handle)
+{
+ struct cma_id_private *listen_id_priv, *id_priv;
+ int ret;
+
+ listen_id_priv = container_of(event->id, struct cma_id_private, id);
+ id_priv = ucma_alloc_id(event->id->context);
+ if (!id_priv) {
+ ucma_destroy_kern_id(handle);
+ ret = -ENOMEM;
+ goto err;
+ }
+
+ event->listen_id = event->id;
+ event->id = &id_priv->id;
+ id_priv->handle = handle;
+
+ ret = ucma_query_route(&id_priv->id);
+ if (ret) {
+ rdma_destroy_id(&id_priv->id);
+ goto err;
+ }
+
+ return 0;
+err:
+ ucma_complete_event(listen_id_priv);
+ return ret;
+}
+
+static int ucma_process_conn_resp(struct cma_id_private *id_priv)
+{
+ struct ucma_abi_accept *cmd;
+ void *msg;
+ int ret, size;
+
+ ret = ucma_modify_qp_rtr(&id_priv->id);
+ if (ret)
+ goto err;
+
+ ret = ucma_modify_qp_rts(&id_priv->id);
+ if (ret)
+ goto err;
+
+ CMA_CREATE_MSG_CMD(msg, cmd, UCMA_CMD_ACCEPT, size);
+ cmd->id = id_priv->handle;
+
+ ret = write(cma_fd, msg, size);
+ if (ret != size) {
+ ret = (ret > 0) ? -ENODATA : ret;
+ goto err;
+ }
+
+ return 0;
+err:
+ ucma_modify_qp_err(&id_priv->id);
+ return ret;
+}
+
+static int ucma_process_establish(struct rdma_cm_id *id)
+{
+ int ret;
+
+ ret = ucma_modify_qp_rts(id);
+ if (ret)
+ ucma_modify_qp_err(id);
+
+ return ret;
+}
+
+int rdma_get_cm_event(struct rdma_cm_event **event)
+{
+ struct ucma_abi_event_resp *resp;
+ struct ucma_abi_get_event *cmd;
+ struct cma_id_private *id_priv;
+ struct rdma_cm_event *evt;
+ void *msg;
+ int ret, size;
+
+ if (!event)
+ return -EINVAL;
+
+ evt = malloc(sizeof *evt + RDMA_MAX_PRIVATE_DATA);
+ if (!evt)
+ return -ENOMEM;
+
+retry:
+ CMA_CREATE_MSG_CMD_RESP(msg, cmd, resp, UCMA_CMD_GET_EVENT, size);
+ ret = write(cma_fd, msg, size);
+ if (ret != size) {
+ ret = (ret > 0) ? -ENODATA : ret;
+ goto err;
+ }
+
+ id_priv = (void *) (uintptr_t) resp->uid;
+ evt->id = &id_priv->id;
+ ucma_copy_event_from_kern(evt, resp);
+
+ switch (evt->event) {
+ case RDMA_CM_EVENT_ADDR_RESOLVED:
+ evt->status = ucma_query_route(&id_priv->id);
+ if (evt->status)
+ evt->event = RDMA_CM_EVENT_ADDR_ERROR;
+ break;
+ case RDMA_CM_EVENT_ROUTE_RESOLVED:
+ evt->status = ucma_query_route(&id_priv->id);
+ if (evt->status)
+ evt->event = RDMA_CM_EVENT_ROUTE_ERROR;
+ break;
+ case RDMA_CM_EVENT_CONNECT_REQUEST:
+ ret = ucma_process_conn_req(evt, resp->id);
+ if (ret)
+ goto retry;
+ break;
+ case RDMA_CM_EVENT_CONNECT_RESPONSE:
+ evt->status = ucma_process_conn_resp(id_priv);
+ if (!evt->status)
+ evt->event = RDMA_CM_EVENT_ESTABLISHED;
+ else
+ evt->event = RDMA_CM_EVENT_CONNECT_ERROR;
+ break;
+ case RDMA_CM_EVENT_ESTABLISHED:
+ evt->status = ucma_process_establish(&id_priv->id);
+ if (evt->status)
+ evt->event = RDMA_CM_EVENT_CONNECT_ERROR;
+ break;
+ case RDMA_CM_EVENT_REJECTED:
+ ucma_modify_qp_err(evt->id);
+ break;
+ default:
+ break;
+ }
+
+ *event = evt;
+ return 0;
+err:
+ free(evt);
+ return ret;
+}
--- /dev/null
+RDMACM_1.0 {
+ global:
+ rdma_create_id;
+ rdma_destroy_id;
+ rdma_bind_addr;
+ rdma_resolve_addr;
+ rdma_resolve_route;
+ rdma_create_qp;
+ rdma_destroy_qp;
+ rdma_connect;
+ rdma_listen;
+ rdma_accept;
+ rdma_reject;
+ rdma_disconnect;
+ rdma_get_cm_event;
+ rdma_ack_cm_event;
+ local: *;
+};
projects / ~shefty / librdmacm.git / commitdiff