From 6f476e59fb0d21824643039a0679e24a07d4733c Mon Sep 17 00:00:00 2001 From: Hal Rosenstock Date: Thu, 27 Jun 2013 21:48:11 +0300 Subject: [PATCH] Add support for preloading IPv4 and IPv6 ACM caches A hosts file (ibacm_hosts.cfg) is configured with tuples of IP addresses (either IPv4 or IPv6) and IB GID. This information in conjunction with the path record preloading allows for the IPv4 and IPv6 ACM caches to be preloaded. IPv4 was tested with both rping and ucmatose as well as acme. IPv6 was tested with ucmatose and acme. The remaining ACM cache to be preloaded is the name cache. This will be added in subsequent patch. Signed-off-by: Hal Rosenstock Signed-off-by: Sean Hefty --- Makefile.am | 2 +- acm_notes.txt | 4 ++ ibacm_hosts.cfg | 12 ++++++ man/ibacm.1 | 4 ++ src/acm.c | 103 ++++++++++++++++++++++++++++++++++++++++++++++++ src/acme.c | 16 ++++++++ 6 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 ibacm_hosts.cfg diff --git a/Makefile.am b/Makefile.am index a0274f7..0de7799 100644 --- a/Makefile.am +++ b/Makefile.am @@ -19,7 +19,7 @@ man_MANS = \ man/ibacm.7 EXTRA_DIST = src/acm_mad.h src/libacm.h ibacm.init.in \ - linux/osd.h linux/dlist.h ibacm.spec.in $(man_MANS) + linux/osd.h linux/dlist.h ibacm.spec.in $(man_MANS) ibacm_hosts.cfg install-exec-hook: install -D -m 755 ibacm.init $(DESTDIR)$(sysconfdir)/init.d/ibacm; diff --git a/acm_notes.txt b/acm_notes.txt index fac0920..3903b8a 100644 --- a/acm_notes.txt +++ b/acm_notes.txt @@ -135,3 +135,7 @@ be set to opensm_full_v1 for this file format. Default format is none which does not preload these caches. See dump_pr.notes.txt in dump_pr for more information on the opensm_full_v1 file format and how to configure OpenSM to generate this file. + +Additional IPv4 and IPv6 caches can be be preloaded by using the +addr_preload option. The default is "none" which does not preload these +caches. diff --git a/ibacm_hosts.cfg b/ibacm_hosts.cfg new file mode 100644 index 0000000..9011691 --- /dev/null +++ b/ibacm_hosts.cfg @@ -0,0 +1,12 @@ +# InfiniBand Communication Management Assistant for clusters hosts file +# +# Entry format is: +# address IB GID +# +# address - IPv4 or IPv6 formatted address +# +# There can be multiple entries for a single IB GID +# +# Samples: +# 192.168.1.3 fe80::8:f104:39a:169 +# fe80::208:f104:39a:169 fe80::8:f104:39a:169 diff --git a/man/ibacm.1 b/man/ibacm.1 index ec97b35..2ca7eec 100644 --- a/man/ibacm.1 +++ b/man/ibacm.1 @@ -156,5 +156,9 @@ format. Default format is none which does not preload these caches. See dump_pr.notes.txt in dump_pr for more information on the full_opensm_v1 file format and how to configure OpenSM to generate this file. +.P +Additional IPv4 and IPv6 caches can be be preloaded by using the +hosts_file option. The default is "none" which does not preload these +caches. .SH "SEE ALSO" ibacm(7), ib_acme(1), rdma_cm(7) diff --git a/src/acm.c b/src/acm.c index a37ebaa..f55ebc0 100644 --- a/src/acm.c +++ b/src/acm.c @@ -82,6 +82,11 @@ enum acm_route_preload { ACM_ROUTE_PRELOAD_OSM_FULL_V1 }; +enum acm_addr_preload { + ACM_ADDR_PRELOAD_NONE, + ACM_ADDR_PRELOAD_HOSTS +}; + /* * Nested locking order: dest -> ep, dest -> port */ @@ -219,6 +224,7 @@ static char *acme = BINDIR "/ib_acme -A"; static char *opts_file = ACM_CONF_DIR "/" ACM_OPTS_FILE; static char *addr_file = ACM_CONF_DIR "/" ACM_ADDR_FILE; static char route_data_file[128] = ACM_CONF_DIR "/ibacm_route.data"; +static char addr_data_file[128] = ACM_CONF_DIR "/ibacm_hosts.data"; static char log_file[128] = "/var/log/ibacm.log"; static int log_level = 0; static char lock_file[128] = "/var/run/ibacm.pid"; @@ -237,6 +243,7 @@ static int recv_depth = 1024; static uint8_t min_mtu = IBV_MTU_2048; static uint8_t min_rate = IBV_RATE_10_GBPS; static enum acm_route_preload route_preload; +static enum acm_addr_preload addr_preload; #define acm_log(level, format, ...) \ acm_write(level, "%s: "format, __func__, ## __VA_ARGS__) @@ -2464,6 +2471,16 @@ static enum acm_route_preload acm_convert_route_preload(char *param) return route_preload; } +static enum acm_route_preload acm_convert_addr_preload(char *param) +{ + if (!stricmp("none", param) || !stricmp("no", param)) + return ACM_ADDR_PRELOAD_NONE; + else if (!stricmp("acm_hosts", param)) + return ACM_ADDR_PRELOAD_HOSTS; + + return addr_preload; +} + static enum ibv_rate acm_get_rate(uint8_t width, uint8_t speed) { switch (width) { @@ -2788,6 +2805,73 @@ err: return ret; } +static void acm_parse_hosts_file(struct acm_ep *ep) +{ + FILE *f; + char s[120]; + char addr[INET6_ADDRSTRLEN], gid[INET6_ADDRSTRLEN]; + uint8_t name[ACM_MAX_ADDRESS]; + struct in6_addr ip_addr, ib_addr; + struct acm_dest *dest, *new_dest; + uint8_t addr_type; + + if (!(f = fopen(addr_data_file, "r"))) { + acm_log(0, "ERROR - couldn't open %s\n", addr_data_file); + return; + } + + while (fgets(s, sizeof s, f)) { + if (s[0] == '#') + continue; + + if (sscanf(s, "%46s%46s", addr, gid) != 2) + continue; + + acm_log(2, "%s", s); + if (inet_pton(AF_INET6, gid, &ib_addr) <= 0) { + acm_log(0, "ERROR - %s is not IB GID\n", gid); + continue; + } + if (inet_pton(AF_INET, addr, &ip_addr) > 0) + addr_type = ACM_ADDRESS_IP; + else if (inet_pton(AF_INET6, addr, &ip_addr) > 0) + addr_type = ACM_ADDRESS_IP6; + else { + acm_log(0, "ERROR - %s is not IP address\n", addr); + continue; + } + + memset(name, 0, ACM_MAX_ADDRESS); + memcpy(name, &ib_addr, sizeof(ib_addr)); + dest = acm_get_dest(ep, ACM_ADDRESS_GID, name); + if (!dest) { + acm_log(0, "ERROR - IB GID %s not found in cache\n", gid); + continue; + } + + memset(name, 0, ACM_MAX_ADDRESS); + if (addr_type == ACM_ADDRESS_IP) + memcpy(name, &ip_addr, 4); + else + memcpy(name, &ip_addr, sizeof(ip_addr)); + new_dest = acm_acquire_dest(ep, addr_type, name); + if (!new_dest) { + acm_log(0, "ERROR - unable to create dest %s\n", addr); + continue; + } + new_dest->path = dest->path; + new_dest->remote_qpn = dest->remote_qpn; + new_dest->addr_timeout = dest->addr_timeout; + new_dest->route_timeout = dest->route_timeout; + new_dest->state = dest->state; + acm_put_dest(new_dest); + acm_log(1, "added host %s address type %d IB GID %s\n", + addr, addr_type, gid); + } + + fclose(f); +} + static int acm_assign_ep_names(struct acm_ep *ep) { FILE *faddr; @@ -2856,6 +2940,11 @@ static int acm_assign_ep_names(struct acm_ep *ep) return !index; } +/* + * We currently require that the routing data be preloaded in order to + * load the address data. This is backwards from normal operation, which + * usually resolves the address before the route. + */ static void acm_ep_preload(struct acm_ep *ep) { switch (route_preload) { @@ -2863,6 +2952,14 @@ static void acm_ep_preload(struct acm_ep *ep) if (acm_parse_osm_fullv1(ep)) acm_log(0, "ERROR - failed to preload EP\n"); break; + default: + return; + } + + switch (addr_preload) { + case ACM_ADDR_PRELOAD_HOSTS: + acm_parse_hosts_file(ep); + break; default: break; } @@ -3366,6 +3463,10 @@ static void acm_set_options(void) route_preload = acm_convert_route_preload(value); else if (!stricmp("route_data_file", opt)) strcpy(route_data_file, value); + else if (!stricmp("addr_preload", opt)) + addr_preload = acm_convert_addr_preload(value); + else if (!stricmp("addr_data_file", opt)) + strcpy(addr_data_file, opt); } fclose(f); @@ -3391,6 +3492,8 @@ static void acm_log_options(void) acm_log(0, "minimum rate %d\n", min_rate); acm_log(0, "route preload %d\n", route_preload); acm_log(0, "route data file %s\n", route_data_file); + acm_log(0, "address preload %d\n", addr_preload); + acm_log(0, "address data file %s\n", addr_data_file); } static FILE *acm_open_log(void) diff --git a/src/acme.c b/src/acme.c index 0f1c53a..cb2128c 100644 --- a/src/acme.c +++ b/src/acme.c @@ -260,6 +260,22 @@ static void gen_opts_temp(FILE *f) fprintf(f, "# Default is ACM_CONF_DIR/ibacm_route.data\n"); fprintf(f, "# /etc/rdma/ibacm_route.data\n"); fprintf(f, "\n"); + fprintf(f, "# addr_preload:\n"); + fprintf(f, "# Specifies if the ACM address cache should be preloaded, or built on demand.\n"); + fprintf(f, "# If preloaded, indicates the method used to build the cache.\n"); + fprintf(f, "# Supported preload values are:\n"); + fprintf(f, "# none - The address cache is not pre-built (default)\n"); + fprintf(f, "# acm_hosts - ACM address to GID file format\n"); + fprintf(f, "\n"); + fprintf(f, "addr_preload none\n"); + fprintf(f, "\n"); + fprintf(f, "# addr_data_file:\n"); + fprintf(f, "# Specifies the location of the address data file to use when preloading\n"); + fprintf(f, "# the ACM cache. This option is only valid if addr_preload\n"); + fprintf(f, "# indicates that address data should be read from a file.\n"); + fprintf(f, "# Default is ACM_CONF_DIR/ibacm_hosts.data\n"); + fprintf(f, "# /etc/rdma/ibacm_hosts.data\n"); + fprintf(f, "\n"); } static int open_dir(void) -- 2.46.0