Add support for preloading IPv4 and IPv6 ACM caches

A hosts file (ibacm_hosts.cfg) is configured with tuples of
IP addresses (either IPv4 or IPv6) and IB GID. This information
in conjunction with the path record preloading allows
for the IPv4 and IPv6 ACM caches to be preloaded.

IPv4 was tested with both rping and ucmatose as well as
acme. IPv6 was tested with ucmatose and acme.

The remaining ACM cache to be preloaded is the name cache.
This will be added in subsequent patch.

Signed-off-by: Hal Rosenstock <hal@mellanox.com>
Signed-off-by: Sean Hefty <sean.hefty@intel.com>

diff --git a/Makefile.am b/Makefile.am
index a0274f710a5d3a0ab708605277f6d8bd3c762ee7..0de7799ffed4d1b983cc5631b827faf902813662 100644 (file)
--- a/Makefile.am
+++ b/Makefile.am
@@ -19,7 +19,7 @@ man_MANS = \
        man/ibacm.7
 
 EXTRA_DIST = src/acm_mad.h src/libacm.h ibacm.init.in \
-            linux/osd.h linux/dlist.h ibacm.spec.in $(man_MANS)
+            linux/osd.h linux/dlist.h ibacm.spec.in $(man_MANS) ibacm_hosts.cfg
 
 install-exec-hook:
        install -D -m 755 ibacm.init $(DESTDIR)$(sysconfdir)/init.d/ibacm;

diff --git a/acm_notes.txt b/acm_notes.txt
index fac09201ac1f27a5460335be9d9a73a6766e0382..3903b8ab8335f66ac0bcebfa8319c92485477178 100644 (file)
--- a/acm_notes.txt
+++ b/acm_notes.txt
@@ -135,3 +135,7 @@ be set to opensm_full_v1 for this file format.  Default format is
 none which does not preload these caches.  See dump_pr.notes.txt in dump_pr
 for more information on the opensm_full_v1 file format and how to configure
 OpenSM to generate this file.
+
+Additional IPv4 and IPv6 caches can be be preloaded by using the
+addr_preload option.  The default is "none" which does not preload these
+caches.

diff --git a/ibacm_hosts.cfg b/ibacm_hosts.cfg
new file mode 100644 (file)
index 0000000..9011691
--- /dev/null
+++ b/ibacm_hosts.cfg
@@ -0,0 +1,12 @@
+# InfiniBand Communication Management Assistant for clusters hosts file
+#
+# Entry format is:
+# address IB GID
+#
+# address - IPv4 or IPv6 formatted address
+#
+# There can be multiple entries for a single IB GID
+#
+# Samples:
+# 192.168.1.3 fe80::8:f104:39a:169
+# fe80::208:f104:39a:169 fe80::8:f104:39a:169

diff --git a/man/ibacm.1 b/man/ibacm.1
index ec97b35c1e45471cba0c99fd49759e846ee4921a..2ca7eec24bd6b7ba58e2eb36c07b2941654bc30e 100644 (file)
--- a/man/ibacm.1
+++ b/man/ibacm.1
@@ -156,5 +156,9 @@ format.  Default format is none which does not preload these caches.
 See dump_pr.notes.txt in dump_pr for more information on the
 full_opensm_v1 file format and how to configure OpenSM to
 generate this file.
+.P
+Additional IPv4 and IPv6 caches can be be preloaded by using the
+hosts_file option.  The default is "none" which does not preload these
+caches.
 .SH "SEE ALSO"
 ibacm(7), ib_acme(1), rdma_cm(7)

diff --git a/src/acm.c b/src/acm.c
index a37ebaa19b35422222906745ffa5a71531934f2a..a57ab17313fcb93cc708d91a9cbcb6da742c8eee 100644 (file)
--- a/src/acm.c
+++ b/src/acm.c
@@ -82,6 +82,11 @@ enum acm_route_preload {
        ACM_ROUTE_PRELOAD_OSM_FULL_V1
 };
 
+enum acm_addr_preload {
+       ACM_ADDR_PRELOAD_NONE,
+       ACM_ADDR_PRELOAD_HOSTS
+};
+
 /*
  * Nested locking order: dest -> ep, dest -> port
  */
@@ -219,6 +224,7 @@ static char *acme = BINDIR "/ib_acme -A";
 static char *opts_file = ACM_CONF_DIR "/" ACM_OPTS_FILE;
 static char *addr_file = ACM_CONF_DIR "/" ACM_ADDR_FILE;
 static char route_data_file[128] = ACM_CONF_DIR "/ibacm_route.data";
+static char addr_data_file[128] = ACM_CONF_DIR "/ibacm_hosts.data";
 static char log_file[128] = "/var/log/ibacm.log";
 static int log_level = 0;
 static char lock_file[128] = "/var/run/ibacm.pid";
@@ -237,6 +243,7 @@ static int recv_depth = 1024;
 static uint8_t min_mtu = IBV_MTU_2048;
 static uint8_t min_rate = IBV_RATE_10_GBPS;
 static enum acm_route_preload route_preload;
+static enum acm_addr_preload addr_preload;
 
 #define acm_log(level, format, ...) \
        acm_write(level, "%s: "format, __func__, ## __VA_ARGS__)
@@ -2464,6 +2471,16 @@ static enum acm_route_preload acm_convert_route_preload(char *param)
        return route_preload;
 }
 
+static enum acm_route_preload acm_convert_addr_preload(char *param)
+{
+       if (!stricmp("none", param) || !stricmp("no", param))
+               return ACM_ADDR_PRELOAD_NONE;
+       else if (!stricmp("acm_hosts", param))
+               return ACM_ADDR_PRELOAD_HOSTS;
+
+       return addr_preload;
+}
+
 static enum ibv_rate acm_get_rate(uint8_t width, uint8_t speed)
 {
        switch (width) {
@@ -2788,6 +2805,74 @@ err:
        return ret;
 }
 
+static void acm_parse_hosts_file(struct acm_ep *ep)
+{
+       FILE *f;
+       char s[120];
+       char addr[INET6_ADDRSTRLEN], gid[INET6_ADDRSTRLEN];
+       uint8_t name[ACM_MAX_ADDRESS];
+       struct in6_addr ip_addr, ib_addr;
+       struct acm_dest *dest, *new_dest;
+       uint8_t addr_type;
+
+       if (!(f = fopen(addr_data_file, "r"))) {
+               acm_log(0, "ERROR - couldn't open %s\n", addr_data_file);
+               return;
+        }
+
+       while (fgets(s, sizeof s, f)) {
+               if (s[0] == '#')
+                       continue;
+
+               if (sscanf(s, "%46s%46s", addr, gid) != 2)
+                       continue;
+
+               acm_log(2, "%s", s);
+               if (inet_pton(AF_INET6, gid, &ib_addr) <= 0) {
+                       acm_log(0, "ERROR - %s is not IB GID\n", gid);
+                       continue;
+               }
+               if (inet_pton(AF_INET, addr, &ip_addr) > 0)
+                       addr_type = ACM_ADDRESS_IP;
+               else if (inet_pton(AF_INET6, addr, &ip_addr) > 0)
+                       addr_type = ACM_ADDRESS_IP6;
+               else {
+                       acm_log(0, "ERROR - %s is not IP address\n", addr);
+                       continue;
+               }
+
+               memset(name, 0, ACM_MAX_ADDRESS);
+               memcpy(name, &ib_addr, sizeof(ib_addr));
+               dest = acm_get_dest(ep, ACM_ADDRESS_GID, name);
+               if (!dest) {
+                       acm_log(0, "ERROR - IB GID %s not found in cache\n", gid);
+                       continue;
+               }
+
+               memset(name, 0, ACM_MAX_ADDRESS);
+               if (addr_type == ACM_ADDRESS_IP)
+                       memcpy(name, &ip_addr, 4);
+               else
+                       memcpy(name, &ip_addr, sizeof(ip_addr));
+               new_dest = acm_acquire_dest(ep, addr_type, name);
+               if (!new_dest) {
+                       acm_log(0, "ERROR - unable to create dest %s\n", addr);
+                       continue;
+               }
+               new_dest->path = dest->path;
+               new_dest->remote_qpn = dest->remote_qpn;
+               new_dest->addr_timeout = dest->addr_timeout;
+               new_dest->route_timeout = dest->route_timeout;
+               new_dest->state = dest->state;
+               acm_put_dest(new_dest);
+               acm_put_dest(dest);
+               acm_log(1, "added host %s address type %d IB GID %s\n",
+                       addr, addr_type, gid);
+       }
+
+       fclose(f);
+}
+
 static int acm_assign_ep_names(struct acm_ep *ep)
 {
        FILE *faddr;
@@ -2856,6 +2941,11 @@ static int acm_assign_ep_names(struct acm_ep *ep)
        return !index;
 }
 
+/*
+ * We currently require that the routing data be preloaded in order to
+ * load the address data.  This is backwards from normal operation, which
+ * usually resolves the address before the route.
+ */
 static void acm_ep_preload(struct acm_ep *ep)
 {
        switch (route_preload) {
@@ -2863,6 +2953,14 @@ static void acm_ep_preload(struct acm_ep *ep)
                if (acm_parse_osm_fullv1(ep))
                        acm_log(0, "ERROR - failed to preload EP\n");
                break;
+       default:
+               return;
+       }
+
+       switch (addr_preload) {
+       case ACM_ADDR_PRELOAD_HOSTS:
+               acm_parse_hosts_file(ep);
+               break;
        default:
                break;
        }
@@ -3366,6 +3464,10 @@ static void acm_set_options(void)
                        route_preload = acm_convert_route_preload(value);
                else if (!stricmp("route_data_file", opt))
                        strcpy(route_data_file, value);
+               else if (!stricmp("addr_preload", opt))
+                       addr_preload = acm_convert_addr_preload(value);
+               else if (!stricmp("addr_data_file", opt))
+                       strcpy(addr_data_file, value);
        }
 
        fclose(f);
@@ -3391,6 +3493,8 @@ static void acm_log_options(void)
        acm_log(0, "minimum rate %d\n", min_rate);
        acm_log(0, "route preload %d\n", route_preload);
        acm_log(0, "route data file %s\n", route_data_file);
+       acm_log(0, "address preload %d\n", addr_preload);
+       acm_log(0, "address data file %s\n", addr_data_file);
 }
 
 static FILE *acm_open_log(void)

diff --git a/src/acme.c b/src/acme.c
index a0dcf8d8ae32d26548e539dc90c7ce9846d5875f..7f93b683c411ed715e39a7ecfaaae4286d1f6f33 100644 (file)
--- a/src/acme.c
+++ b/src/acme.c
@@ -260,6 +260,22 @@ static void gen_opts_temp(FILE *f)
        fprintf(f, "# Default is ACM_CONF_DIR/ibacm_route.data\n");
        fprintf(f, "# route_data_file /etc/rdma/ibacm_route.data\n");
        fprintf(f, "\n");
+       fprintf(f, "# addr_preload:\n");
+       fprintf(f, "# Specifies if the ACM address cache should be preloaded, or built on demand.\n");
+       fprintf(f, "# If preloaded, indicates the method used to build the cache.\n");
+       fprintf(f, "# Supported preload values are:\n");
+       fprintf(f, "# none - The address cache is not pre-built (default)\n");
+       fprintf(f, "# acm_hosts - ACM address to GID file format\n");
+       fprintf(f, "\n");
+       fprintf(f, "addr_preload none\n");
+       fprintf(f, "\n");
+       fprintf(f, "# addr_data_file:\n");
+       fprintf(f, "# Specifies the location of the address data file to use when preloading\n");
+       fprintf(f, "# the ACM cache.  This option is only valid if addr_preload\n");
+       fprintf(f, "# indicates that address data should be read from a file.\n");
+       fprintf(f, "# Default is ACM_CONF_DIR/ibacm_hosts.data\n");
+       fprintf(f, "# addr_data_file /etc/rdma/ibacm_hosts.data\n");
+       fprintf(f, "\n");
 }
 
 static int open_dir(void)