From ebc3c521d85ebc9ccb5fbaa4c0dad7ec4ec2c41a Mon Sep 17 00:00:00 2001 From: Hal Rosenstock Date: Wed, 15 Jun 2005 15:22:39 +0000 Subject: [PATCH] r2616: Initial version of user AT library (and example) Signed-off-by: Hal Rosenstock --- AUTHORS | 2 +- COPYING | 2 +- configure.in | 8 +- examples/att.c | 199 +++++++++++++++++++++++ include/infiniband/at.h | 239 ++++++++++++++++++++++++++++ include/infiniband/at_abi.h | 171 ++++++++++++++++++++ include/infiniband/sa.h | 36 +++++ libibat.spec.in | 40 +++++ src/at.c | 306 ++++++++++++++++++++++++++++++++++++ src/libibat.map | 11 ++ 10 files changed, 1008 insertions(+), 6 deletions(-) create mode 100644 examples/att.c create mode 100644 include/infiniband/at.h create mode 100644 include/infiniband/at_abi.h create mode 100644 libibat.spec.in create mode 100644 src/at.c create mode 100644 src/libibat.map diff --git a/AUTHORS b/AUTHORS index c79db95..16259f8 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1 +1 @@ -Libor Michalek +Hal Rosenstock diff --git a/COPYING b/COPYING index ee1a79f..db8bc99 100644 --- a/COPYING +++ b/COPYING @@ -3,7 +3,7 @@ licenses. You may choose to be licensed under the terms of the the OpenIB.org BSD license or the GNU General Public License (GPL) Version 2, both included below. -Copyright (c) 2004 Topspin Communications. All rights reserved. +Copyright (c) 2005 Voltaire, Inc. All rights reserved. ================================================================== diff --git a/configure.in b/configure.in index 2eb4bee..2044412 100644 --- a/configure.in +++ b/configure.in @@ -1,11 +1,11 @@ dnl Process this file with autoconf to produce a configure script. AC_PREREQ(2.57) -AC_INIT(libibcm, 0.9.0, openib-general@openib.org) -AC_CONFIG_SRCDIR([src/cm.c]) +AC_INIT(libibat, 0.9.0, openib-general@openib.org) +AC_CONFIG_SRCDIR([src/at.c]) AC_CONFIG_AUX_DIR(config) AM_CONFIG_HEADER(config.h) -AM_INIT_AUTOMAKE(libibcm, 0.9.0) +AM_INIT_AUTOMAKE(libibat, 0.9.0) AC_DISABLE_STATIC AM_PROG_LIBTOOL @@ -32,5 +32,5 @@ AC_CACHE_CHECK(whether ld accepts --version-script, ac_cv_version_script, AM_CONDITIONAL(HAVE_LD_VERSION_SCRIPT, test "$ac_cv_version_script" = "yes") -AC_CONFIG_FILES([Makefile libibcm.spec]) +AC_CONFIG_FILES([Makefile libibat.spec]) AC_OUTPUT diff --git a/examples/att.c b/examples/att.c new file mode 100644 index 0000000..115e461 --- /dev/null +++ b/examples/att.c @@ -0,0 +1,199 @@ +/* + * Copyright (c) 2005 Voltaire, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * $Id:$ + */ + +#if HAVE_CONFIG_H +# include +#endif /* HAVE_CONFIG_H */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#if __BYTE_ORDER == __LITTLE_ENDIAN +static inline uint64_t cpu_to_be64(uint64_t x) { return bswap_64(x); } +static inline uint32_t cpu_to_be32(uint32_t x) { return bswap_32(x); } +static inline uint16_t cpu_to_be16(uint16_t x) { return bswap_16(x); } +static inline uint64_t be64_to_cpu(uint64_t x) { return bswap_64(x); } +#define be32_to_cpu ntohl +#define be16_to_cpu ntohs +#elif __BYTE_ORDER == __BIG_ENDIAN +static inline uint64_t cpu_to_be64(uint64_t x) { return x; } +static inline uint32_t cpu_to_be32(uint32_t x) { return x; } +static inline uint16_t cpu_to_be16(uint16_t x) { return x; } +static inline uint64_t be64_to_cpu(uint64_t x) { return x; } +static inline uint32_t be32_to_cpu(uint32_t x) { return x; } +static inline uint16_t be16_to_cpu(uint16_t x) { return x; } +#endif + +//#define WARN(fmt, ...) while (0) {} +//#define DEBUG(fmt, ...) while (0) {} +#define WARN(fmt, arg ...) printf("uatt: " fmt "\n", ## arg); +#define DEBUG(fmt, arg ...) printf("uatt: %s: " fmt "\n", __FUNCTION__, ## arg); + +#define MAX_REQ 32 +#define SLEEP 30 + +static struct ib_sa_path_rec att_path[MAX_REQ]; +static struct ib_at_completion att_path_comp[MAX_REQ]; +static struct ib_at_ib_route att_rt[MAX_REQ]; +static struct ib_at_completion att_rt_comp[MAX_REQ]; + + +static void att_path_comp_fn(uint64_t req_id, void *context, int rec_num) +{ + struct ib_sa_path_rec *path = context; + int i; + + DEBUG("id %lld context %p completed with rec_num %d", + req_id, context, rec_num); + + i = path - att_path; + + if (i < 0 || i >= MAX_REQ) { + DEBUG("bad req context %d\n", i); + return; + } + + if (rec_num <= 0) { + printf("path resolve failed (%d)!!!\n", rec_num); + return; + } + + printf("===> slid 0x%x dlid 0x%x\n", + (int)be16_to_cpu(path->slid), + (int)be16_to_cpu(path->dlid)); +} + +static void dump_rt(struct ib_at_ib_route *rt) +{ + printf("===> rt %p sgid 0x%016llx%016llx dgid 0x%016llx%016llx\n", + rt, + be64_to_cpu(rt->sgid.global.subnet_prefix), + be64_to_cpu(rt->sgid.global.interface_id), + be64_to_cpu(rt->dgid.global.subnet_prefix), + be64_to_cpu(rt->dgid.global.interface_id)); +} + +static void att_rt_comp_fn(uint64_t req_id, void *context, int rec_num) +{ + struct ib_at_ib_route *rt = context; + int r, i; + uint64_t req_id2; + + DEBUG("id %lld context %p completed with rec_num %d", + req_id, context, rec_num); + + i = rt - att_rt; + + if (i < 0 || i >= MAX_REQ) { + DEBUG("bad req context %d\n", i); + return; + } + + if (rec_num <= 0) + return; + + dump_rt(rt); + + DEBUG("ib_at_paths_by_route: route %p context %p compl %p", rt, att_path_comp[i].context, att_path_comp + i); + r = ib_at_paths_by_route(rt, 0, att_path_comp[i].context, + 1, att_path_comp + i, &req_id2); + + DEBUG("ib_at_paths_by_route: returned %d id %lld %lld", + r, att_path_comp[i].req_id, req_id2); + + /* Check for callback events */ + /* Should this be timed ? */ + r = ib_at_callback_get(); + if (r) { + printf("Error <%d:%d> getting callback event\n", + r, errno); + } +} + +int main(int argc, char **argv) +{ + int r, i; + + DEBUG("uat test start"); + + for (i = 0; i < MAX_REQ; i++) { + att_rt_comp[i].fn = att_rt_comp_fn; + att_rt_comp[i].context = att_rt + i; + att_path_comp[i].fn = att_path_comp_fn; + att_path_comp[i].context = att_path + i; + } + + for (i = 0; i < MAX_REQ; i++) { + r = ib_at_route_by_ip(0x0100a8c0, 0, 0, 0, + att_rt + i, att_rt_comp + i); + + DEBUG("ib_at_route_by_ip: ret %d errno %d for request %d id %lld", + r, errno, i + 1, att_rt_comp[i].req_id); + if (r == 1) + att_rt_comp_fn(att_rt_comp[i].req_id, att_rt + i, 1); + } + + /* make sleep period command line driven !!! */ + DEBUG("sleeping for %d secs", SLEEP); + sleep(SLEEP); /* hang out here for a while */ + + DEBUG("uat test cleanup"); + + for (i = 0; i < MAX_REQ; i++) { + if ((r = ib_at_cancel(att_rt_comp[i].req_id)) < 0) { + DEBUG("cancel but no rt id %lld ret %d errno %d", att_rt_comp[i].req_id, r, errno); + } else + DEBUG("canceling rt id %lld ret %d", + att_rt_comp[i].req_id, r); + + if ((r = ib_at_cancel(att_path_comp[i].req_id)) < 0) { + DEBUG("cancel but no path id %lld ret %d errno %d", att_path_comp[i].req_id, r, errno); + } else + DEBUG("canceling path id %lld ret %d", + att_path_comp[i].req_id, r); + } + + return 0; +} diff --git a/include/infiniband/at.h b/include/infiniband/at.h new file mode 100644 index 0000000..e046ecb --- /dev/null +++ b/include/infiniband/at.h @@ -0,0 +1,239 @@ +/* + * Copyright (c) 2004,2005 Voltaire Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * + * $Id$ + */ + +#if !defined( AT_H ) +#define AT_H + +#include +#include + +enum ib_at_multipathing_type { + IB_AT_PATH_SAME_PORT = 0, + IB_AT_PATH_SAME_HCA = 1, /* but different ports if applicable */ + IB_AT_PATH_SAME_SYSTEM = 2, /* but different ports if applicable */ + IB_AT_PATH_INDEPENDENT_HCA = 3, + IB_AT_PATH_SRC_ROUTE = 4, /* application controlled multipathing */ +}; + +enum ib_at_route_flags { + IB_AT_ROUTE_USE_DEFAULTS = 0, + IB_AT_ROUTE_FORCE_ATS = 1, + IB_AT_ROUTE_FORCE_ARP = 2, + IB_AT_ROUTE_FORCE_RESOLVE = 4, +}; + +struct ib_at_path_attr { + uint16_t qos_tag; + uint16_t pkey; + uint8_t multi_path_type; +}; + +struct ib_at_ib_route { + union ibv_gid sgid; + union ibv_gid dgid; + struct ibv_device *out_dev; + int out_port; + struct ib_at_path_attr attr; +}; + +enum ib_at_op_status { + IB_AT_STATUS_INVALID = 0, + IB_AT_STATUS_PENDING = 1, + IB_AT_STATUS_COMPLETED = 2, + IB_AT_STATUS_ERROR = 3, + IB_AT_STATUS_CANCELED = 4, +}; + +/* + * ib_at_completion structure - callback function parameters structure + * @completion: completion call back function + * @context: user defined context pointer + * @req_id: asynchronous request ID - optional, out + * + * The following asynchronous resolution function behavior is as follows: + * If the resolve operation can be fulfilled immediately, then the output + * structures are set and the number of filled structures is returned. + * + * If the resolve operation cannot by fulfilled immediately and + * an ib_at_completion structure is not provided, + * then the function immediately returns -EWOULDBLOCK. + * + * If ib_at_completion structure is provided and an asynchronous + * operation is started, the function immediately returns zero, + * and the request ID field (req_id) is set if the pointer is + * non NULL. This request ID may be used to cancel the operation, + * or to poll its status. + * + * When an asynchronous operation completes (successfully or not), + * the callback function is called, passing the request ID, + * the supplied user context and the number of output structures. + * If the asynchronous operation did not complete, a negative + * error code is return as the 'rec_num'. + * Valid error codes are: + * -EINTR: operation is canceled + * -EIO: request send failed + * -ETIMEOUT: operation timed out + * + * Returned value of zero records means that the resolution process + * completed, but the given address could not be resolved at this time. + */ +struct ib_at_completion { + void (*fn)(uint64_t req_id, void *context, int rec_num); + void *context; + uint64_t req_id; +}; + +/** + * ib_at_route_by_ip - asynchronously resolve ip route to ib route + * @dst_ip: destination ip + * @src_ip: source ip - optional + * @tos: ip type of service + * @flags: ib_at_route_flags + * @ib_route: out structure + * @async_comp: asynchronous callback structure - optional + * + * Resolve the specified dst_ip to a &struct ib_route structure. + * src_ip can be provide to force specific output interface. + * flags can be used to select resolving method; currently IB-ARP or ATS. + * + * See ib_at_completion structure documentation for asynchronous + * operation details. + */ +int ib_at_route_by_ip(uint32_t dst_ip, uint32_t src_ip, int tos, uint16_t flags, + struct ib_at_ib_route *ib_route, + struct ib_at_completion *async_comp); + +/** + * ib_at_paths_by_route - asynchronously resolve ib route to ib path records + * @ib_route: ib route to resolve + * @mpath_type: ib_at_multipathing_type + * @path_arr: SA path record array - out + * @npath: maximal number of paths to return + * @async_comp: asynchronous callback structure - optional + * @req_id: pointer for request ID + * + * Resolve the specified ib_route to a SA path record array. + * Number of returned paths will not exceed npath. + * Multipathing type may be used to obtain redundant paths for APM, + * other failover schemes, bandwidth aggregation or source based routing. + * Note that multipathing request is meaningless unless npath is greater than 1. + * + * Returned ib_route structure includes the recommended pkey and qos_tag for + * this route. + * + * See ib_at_completion structure documentation for asynchronous operation + * details. + */ +int ib_at_paths_by_route(struct ib_at_ib_route *ib_route, uint32_t mpath_type, + struct ib_sa_path_rec *path_arr, int npath, + struct ib_at_completion *async_comp, uint64_t *req_id); + +/** + * ib_at_ips_by_gid - asynchronously resolve GID to IP addresses + * @gid: GID to resolve + * @dst_ips: array of IPs, out + * @nips: number of IP entries in dst_ips array + * @async_comp: asynchronous callback structure - optional + * + * Resolve the gid to IP addresses, but not more than nips. + * This function rely on the IB-ATS mechanism. + * + * See ib_at_completion structure documentation for asynchronous + * operation details. + */ +int ib_at_ips_by_gid(union ibv_gid *gid, uint32_t *dst_ips, int nips, + struct ib_at_completion *async_comp); + +/** + * ib_at_ips_by_subnet - return local IP addresses by IP subnet + * @network: network to resolve - optional + * @netmask: subnet net mask - optional + * @dst_ips: array of IPs, out + * @nips: number of IP entries in dst_ips array + * + * Return local IP addresses matching the network and netmask, + * but not more than nips. + * + * Note that network and netmask as 0x0 or 0xffffffff returns all local IPs. + */ +int ib_at_ips_by_subnet(uint32_t network, uint32_t netmask, + uint32_t *dst_ips, int nips); + +/** + * ib_at_invalidate_paths - invalidate possibly cached paths keyed by ib_route + * @ib_route: paths key - optional + * + * Returns number of invalidated paths. + * If ib_route is NULL, then the entire cache will be flushed. + */ +int ib_at_invalidate_paths(struct ib_at_ib_route *ib_route); + +/** + * ib_at_cancel - cancel possible active asynchronous operation + * @req_id: asynchronous request ID + * + * Return 0 if canceled, -1 if cancel failed (e.g. bad ID) + */ +int ib_at_cancel(uint64_t req_id); + +/** + * ib_at_status - poll asynchronous operation's status + * @req_id: asynchronous request ID ib_at_op_status + * + * Return non-negative ib_at_op_status value, + * or -EINVAL if the request ID is invalid. + */ +int ib_at_status(uint64_t req_id); + +/** + * ib_at_callback_get - Retrieves the next pending AT callback, + * if no callback is pending waits for a callback event. + */ +int ib_at_callback_get(); + +/** + * ib_at_get_fd - Returns the file descriptor which AT uses to + * submit requests and retrieve callback events. + * + * The primary use of the file descriptor is to test for AT readiness + * events. When the AT becomes ready to READ there is a pending event + * ready, and a subsequent call to ib_at_event_get will not block. + * Note: The user should not read or write directly to the AT file + * descriptor, it will likely result in an error or unexpected + * results. + */ +int ib_at_get_fd(void); + +#endif /* IB_AT_H */ diff --git a/include/infiniband/at_abi.h b/include/infiniband/at_abi.h new file mode 100644 index 0000000..d1beefd --- /dev/null +++ b/include/infiniband/at_abi.h @@ -0,0 +1,171 @@ +/* + * Copyright (c) 2005 Voltaire, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * $Id:$ + */ + +#ifndef AT_ABI_H +#define AT_ABI_H + +#include +/* + * This file must be kept in sync with the kernel's version of + * drivers/infiniband/include/ib_user_at.h + */ + +#define IB_USER_AT_ABI_VERSION 1 + +enum { + IB_USER_AT_CMD_ROUTE_BY_IP, + IB_USER_AT_CMD_PATHS_BY_ROUTE, + IB_USER_AT_CMD_IPS_BY_GID, + IB_USER_AT_CMD_IPS_BY_SUBNET, + IB_USER_AT_CMD_INVALIDATE_PATHS, + IB_USER_AT_CMD_CANCEL, + IB_USER_AT_CMD_STATUS, + + IB_USER_AT_CMD_EVENT, +}; + +/* + * command ABI structures. + */ +struct at_abi_cmd_hdr { + __u32 cmd; + __u16 in; + __u16 out; +}; + +struct at_abi_completion { + void (*fn)(__u64 req_id, void *context, int rec_num); + void *context; + __u64 req_id; +}; + +struct at_abi_path_attr { + __u16 qos_tag; + __u16 pkey; + __u8 multi_path_type; +}; + +struct at_abi_path_rec { + __u8 dgid[16]; + __u8 sgid[16]; + __u16 dlid; + __u16 slid; + __u32 raw_traffic; + __u32 flow_label; + __u32 reversible; + __u32 mtu; + __u16 pkey; + __u8 hop_limit; + __u8 traffic_class; + __u8 numb_path; + __u8 sl; + __u8 mtu_selector; + __u8 rate_selector; + __u8 rate; + __u8 packet_life_time_selector; + __u8 packet_life_time; + __u8 preference; +}; + +struct at_abi_ib_route { + union ibv_gid sgid; + union ibv_gid dgid; + struct ibv_device *out_dev; + int out_port; + struct at_abi_path_attr attr; +}; + +struct at_abi_route_by_ip { + __u32 dst_ip; + __u32 src_ip; + int tos; + __u16 flags; + struct at_abi_ib_route *ib_route; + struct at_abi_completion *async_comp; +}; + +struct at_abi_paths_by_route { + struct at_abi_ib_route *ib_route; + __u32 mpath_type; + struct at_abi_path_rec *path_arr; + int npath; + struct at_abi_completion *async_comp; + __u64 response; +}; + +struct at_abi_paths_by_route_resp { + __u64 req_id; +}; + +struct at_abi_ips_by_gid { + union ibv_gid *gid; + __u32 *dst_ips; + int nips; + struct at_abi_completion *async_comp; +}; + +struct at_abi_ips_by_subnet { + __u32 network; + __u32 netmask; + __u32 *dst_ips; + int nips; +}; + +struct at_abi_invalidate_paths { + struct at_abi_ib_route *ib_route; +}; + +struct at_abi_cancel { + __u64 req_id; +}; + +struct at_abi_status { + __u64 req_id; +}; + +/* + * event notification ABI structures. + */ +struct at_abi_event_get { + __u64 response; +}; + +struct at_abi_event_resp { + __u64 callback; + __u64 context; + __u64 req_id; + int rec_num; +}; + +#endif /* AT_ABI_H */ diff --git a/include/infiniband/sa.h b/include/infiniband/sa.h index a5f960e..2b49aa9 100644 --- a/include/infiniband/sa.h +++ b/include/infiniband/sa.h @@ -1,5 +1,6 @@ /* * Copyright (c) 2004 Topspin Communications. All rights reserved. + * Copyright (c) 2005 Voltaire, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -91,4 +92,39 @@ struct ib_sa_path_rec { uint8_t preference; }; +struct ib_sa_mcmember_rec { + union ibv_gid mgid; + union ibv_gid port_gid; + uint32_t qkey; + uint16_t mlid; + uint8_t mtu_selector; + enum ibv_mtu mtu; + uint8_t traffic_class; + uint16_t pkey; + uint8_t rate_selector; + uint8_t rate; + uint8_t packet_life_time_selector; + uint8_t packet_life_time; + uint8_t sl; + uint32_t flow_label; + uint8_t hop_limit; + uint8_t scope; + uint8_t join_state; + int proxy_join; +}; + +struct ib_sa_service_rec { + uint64_t id; + union ibv_gid gid; + uint16_t pkey; + /* uint16_t resv; */ + uint32_t lease; + uint8_t key[16]; + uint8_t name[64]; + uint8_t data8[16]; + uint16_t data16[8]; + uint32_t data32[4]; + uint64_t data64[2]; +}; + #endif /* IB_SA_H */ diff --git a/libibat.spec.in b/libibat.spec.in new file mode 100644 index 0000000..98ff8b2 --- /dev/null +++ b/libibat.spec.in @@ -0,0 +1,40 @@ +# $Id:$ + +%define prefix /usr +%define ver @VERSION@ +%define RELEASE 1 +%define rel %{?CUSTOM_RELEASE} %{!?CUSTOM_RELEASE:%RELEASE} + +Summary: Userspace InfiniBand Address Translation. +Name: libibat +Version: %ver +Release: %rel +Copyright: Dual GPL/BSD +Group: System Environment/Libraries +BuildRoot: %{_tmppath}/%{name}-%{version}-root +Source: http://openib.org/downloads/%{name}-%{version}.tar.gz +Url: http://openib.org/ + +%description +Along with the OpenIB kernel drivers, libibat provides a userspace +InfiniBand Address Translation API. + +%prep +%setup -q + +%build +%configure +make + +%install +make DESTDIR=${RPM_BUILD_ROOT} install +# remove unpackaged files from the buildroot +rm -f $RPM_BUILD_ROOT%{_libdir}/*.la + +%clean +rm -rf $RPM_BUILD_ROOT + +%files +%defattr(-,root,root) +%{_libdir}/libibat*.so.* +%doc AUTHORS COPYING ChangeLog NEWS README diff --git a/src/at.c b/src/at.c new file mode 100644 index 0000000..4b2de33 --- /dev/null +++ b/src/at.c @@ -0,0 +1,306 @@ +/* + * Copyright (c) 2005 Voltaire, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * $Id:$ + */ + +#if HAVE_CONFIG_H +# include +#endif /* HAVE_CONFIG_H */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#define IB_UAT_DEV_PATH "/dev/infiniband/uat" +#define PFX "libuat: " + +#define AT_CREATE_MSG_CMD_RESP(msg, cmd, resp, type, size) \ +do { \ + struct at_abi_cmd_hdr *hdr; \ + \ + size = sizeof(*hdr) + sizeof(*cmd); \ + msg = alloca(size); \ + if (!msg) \ + return -ENOMEM; \ + hdr = msg; \ + cmd = msg + sizeof(*hdr); \ + hdr->cmd = type; \ + hdr->in = sizeof(*cmd); \ + hdr->out = sizeof(*resp); \ + memset(cmd, 0, sizeof(*cmd)); \ + resp = alloca(sizeof(*resp)); \ + if (!resp) \ + return -ENOMEM; \ + cmd->response = (unsigned long)resp;\ +} while (0) + +#define AT_CREATE_MSG_CMD(msg, cmd, type, size) \ +do { \ + struct at_abi_cmd_hdr *hdr; \ + \ + size = sizeof(*hdr) + sizeof(*cmd); \ + msg = alloca(size); \ + if (!msg) \ + return -ENOMEM; \ + hdr = msg; \ + cmd = msg + sizeof(*hdr); \ + hdr->cmd = type; \ + hdr->in = sizeof(*cmd); \ + hdr->out = 0; \ + memset(cmd, 0, sizeof(*cmd)); \ +} while (0) + +static int fd; + +static void __attribute__((constructor)) ib_at_init(void) +{ + fd = open(IB_UAT_DEV_PATH, O_RDWR); + if (fd < 0) + fprintf(stderr, PFX + "Error <%d:%d> couldn't open IB at device <%s>\n", + fd, errno, IB_UAT_DEV_PATH); + +} + +int ib_at_route_by_ip(uint32_t dst_ip, uint32_t src_ip, int tos, uint16_t flags, + struct ib_at_ib_route *ib_route, + struct ib_at_completion *async_comp) +{ + struct at_abi_route_by_ip *cmd; + void *msg; + int result; + int size; + + AT_CREATE_MSG_CMD(msg, cmd, IB_USER_AT_CMD_ROUTE_BY_IP, size); + + cmd->dst_ip = dst_ip; + cmd->src_ip = src_ip; + cmd->tos = tos; + cmd->flags = flags; + cmd->ib_route = (struct at_abi_ib_route *)ib_route; + cmd->async_comp = (struct at_abi_completion *)async_comp; + + result = write(fd, msg, size); + return result; +} + +int ib_at_paths_by_route(struct ib_at_ib_route *ib_route, uint32_t mpath_type, + struct ib_sa_path_rec *path_arr, int npath, + struct ib_at_completion *async_comp, uint64_t *req_id) +{ + struct at_abi_paths_by_route *cmd; + struct at_abi_paths_by_route_resp *resp; + void *msg; + int result; + int size; + + if (!ib_route || !path_arr || !req_id) + return -EINVAL; + + AT_CREATE_MSG_CMD_RESP(msg, cmd, resp, + IB_USER_AT_CMD_PATHS_BY_ROUTE, size); + + cmd->ib_route = (struct at_abi_ib_route *)ib_route; + cmd->mpath_type = mpath_type; + cmd->path_arr = (struct at_abi_path_rec *)path_arr; + cmd->npath = npath; + cmd->async_comp = (struct at_abi_completion *)async_comp; + + result = write(fd, msg, size); + *req_id = resp->req_id; + return result; +} + +int ib_at_ips_by_gid(union ibv_gid *gid, uint32_t *dst_ips, int nips, + struct ib_at_completion *async_comp) +{ + struct at_abi_ips_by_gid *cmd; + void *msg; + int result; + int size; + + if (!gid || !dst_ips) + return -EINVAL; + + AT_CREATE_MSG_CMD(msg, cmd, IB_USER_AT_CMD_IPS_BY_GID, size); + + memcpy(cmd->gid, gid, sizeof(*cmd->gid)); + cmd->dst_ips = dst_ips; + cmd->nips = nips; + cmd->async_comp = (struct at_abi_completion *)async_comp; + + result = write(fd, msg, size); + return result; +} + +int ib_at_ips_by_subnet(uint32_t network, uint32_t netmask, + uint32_t *dst_ips, int nips) +{ + struct at_abi_ips_by_subnet *cmd; + void *msg; + int result; + int size; + + if (!dst_ips) + return -EINVAL; + + AT_CREATE_MSG_CMD(msg, cmd, IB_USER_AT_CMD_IPS_BY_SUBNET, size); + + cmd->network = network; + cmd->netmask = netmask; + cmd->dst_ips = dst_ips; + cmd->nips = nips; + + result = write(fd, msg, size); + return result; +} + +int ib_at_invalidate_paths(struct ib_at_ib_route *ib_route) +{ + struct at_abi_invalidate_paths *cmd; + void *msg; + int result; + int size; + + if (!ib_route) + return -EINVAL; + + AT_CREATE_MSG_CMD(msg, cmd, IB_USER_AT_CMD_INVALIDATE_PATHS, size); + + cmd->ib_route = (struct at_abi_ib_route *)ib_route; + + result = write(fd, msg, size); + return result; +} + +int ib_at_cancel(uint64_t req_id) +{ + struct at_abi_cancel *cmd; + void *msg; + int result; + int size; + + AT_CREATE_MSG_CMD(msg, cmd, IB_USER_AT_CMD_CANCEL, size); + + cmd->req_id = req_id; + + result = write(fd, msg, size); + return result; +} + +int ib_at_status(uint64_t req_id) +{ + struct at_abi_status *cmd; + void *msg; + int result; + int size; + + AT_CREATE_MSG_CMD(msg, cmd, IB_USER_AT_CMD_STATUS, size); + + cmd->req_id = req_id; + + result = write(fd, msg, size); + return result; +} + +/* + * event processing + */ +int ib_at_callback_get() +{ + struct at_abi_cmd_hdr *hdr; + struct at_abi_event_get *cmd; + struct at_abi_event_resp *resp; + void *msg; + void (*callback)(__u64 req_id, void *context, int rec_num); + int result = 0; + int size; + + size = sizeof(*hdr) + sizeof(*cmd); + msg = alloca(size); + if (!msg) + return -ENOMEM; + + hdr = msg; + cmd = msg + sizeof(*hdr); + + hdr->cmd = IB_USER_AT_CMD_EVENT; + hdr->in = sizeof(*cmd); + hdr->out = sizeof(*resp); + + resp = alloca(sizeof(*resp)); + if (!resp) + return -ENOMEM; + + cmd->response = (unsigned long)resp; + + result = write(fd, msg, size); + + /* + * callback event. + */ + callback = (void *)(unsigned long)resp->callback; + callback(resp->req_id, (void *)(unsigned long)resp->context, resp->rec_num); + + result = 0; + return result; +} + +int ib_at_get_fd(void) +{ + return fd; +} + +int ib_at_callback_get_timed(int timeout_ms) +{ + struct pollfd ufds; + int result; + + ufds.fd = ib_at_get_fd(); + ufds.events = POLLIN; + ufds.revents = 0; + + result = poll(&ufds, 1, timeout_ms); + if (!result) + return -ETIMEDOUT; + + return ib_at_callback_get(); +} diff --git a/src/libibat.map b/src/libibat.map new file mode 100644 index 0000000..65d5ef0 --- /dev/null +++ b/src/libibat.map @@ -0,0 +1,11 @@ +IBAT_1.0 { + global: + ib_at_route_by_ip; + ib_at_paths_by_route; + ib_at_ips_by_gid; + ib_at_ips_by_subnet; + ib_at_invalidate_paths; + ib_at_cancel; + ib_at_status; + local: *; +}; -- 2.46.0