/*
* Copyright (c) 2009-2012 Intel Corporation. All rights reserved.
+ * Copyright (c) 2013 Mellanox Technologies LTD. All rights reserved.
*
* This software is available to you under the OpenIB.org BSD license
* below:
#define src_out data[0]
+#define IB_LID_MCAST_START 0xc000
+
#define MAX_EP_ADDR 4
#define MAX_EP_MC 2
ACM_LOOPBACK_PROT_LOCAL
};
+enum acm_route_preload {
+ ACM_ROUTE_PRELOAD_NONE,
+ ACM_ROUTE_PRELOAD_OSM_FULL_V1
+};
+
/*
* Nested locking order: dest -> ep, dest -> port
*/
static atomic_t counter[ACM_MAX_COUNTER];
/*
- * Service options - may be set through acm_opts file.
+ * Service options - may be set through ibacm_opts.cfg file.
*/
static char *acme = BINDIR "/ib_acme -A";
static char *opts_file = ACM_CONF_DIR "/" ACM_OPTS_FILE;
static char *addr_file = ACM_CONF_DIR "/" ACM_ADDR_FILE;
+static char route_data_file[128] = ACM_CONF_DIR "/ibacm_route.data";
static char log_file[128] = "/var/log/ibacm.log";
static int log_level = 0;
static char lock_file[128] = "/var/run/ibacm.pid";
static int recv_depth = 1024;
static uint8_t min_mtu = IBV_MTU_2048;
static uint8_t min_rate = IBV_RATE_10_GBPS;
+static enum acm_route_preload route_preload;
#define acm_log(level, format, ...) \
acm_write(level, "%s: "format, __func__, ## __VA_ARGS__)
return loopback_prot;
}
+static enum acm_route_preload acm_convert_route_preload(char *param)
+{
+ if (!stricmp("none", param) || !stricmp("no", param))
+ return ACM_ROUTE_PRELOAD_NONE;
+ else if (!stricmp("opensm_full_v1", param))
+ return ACM_ROUTE_PRELOAD_OSM_FULL_V1;
+
+ return route_preload;
+}
+
static enum ibv_rate acm_get_rate(uint8_t width, uint8_t speed)
{
switch (width) {
return fopen(addr_file, "r");
}
+/* Parse "opensm full v1" file to build LID to GUID table */
+static void acm_parse_osm_fullv1_lid2guid(FILE *f, uint64_t *lid2guid)
+{
+ char s[128];
+ char *p, *ptr, *p_guid, *p_lid;
+ uint64_t guid;
+ uint16_t lid;
+
+ while (fgets(s, sizeof s, f)) {
+ if (s[0] == '#')
+ continue;
+ if (!(p = strtok_r(s, " \n", &ptr)))
+ continue; /* ignore blank lines */
+
+ if (strncmp(p, "Switch", sizeof("Switch") - 1) &&
+ strncmp(p, "Channel", sizeof("Channel") - 1) &&
+ strncmp(p, "Router", sizeof("Router") - 1))
+ continue;
+
+ if (!strncmp(p, "Channel", sizeof("Channel") - 1)) {
+ p = strtok_r(NULL, " ", &ptr); /* skip 'Adapter' */
+ if (!p)
+ continue;
+ }
+
+ p_guid = strtok_r(NULL, ",", &ptr);
+ if (!p_guid)
+ continue;
+
+ guid = (uint64_t) strtoull(p_guid, NULL, 16);
+
+ ptr = strstr(ptr, "base LID");
+ if (!ptr)
+ continue;
+ ptr += sizeof("base LID");
+ p_lid = strtok_r(NULL, ",", &ptr);
+ if (!p_lid)
+ continue;
+
+ lid = (uint16_t) strtoul(p_lid, NULL, 0);
+ if (lid >= IB_LID_MCAST_START)
+ continue;
+ if (lid2guid[lid])
+ acm_log(0, "ERROR - duplicate lid %u\n", lid);
+ else
+ lid2guid[lid] = htonll(guid);
+ }
+}
+
+/* Parse 'opensm full v1' file to populate PR cache */
+static int acm_parse_osm_fullv1_paths(FILE *f, uint64_t *lid2guid, struct acm_ep *ep)
+{
+ union ibv_gid sgid, dgid;
+ struct ibv_port_attr attr = { 0 };
+ struct acm_dest *dest;
+ char s[128];
+ char *p, *ptr, *p_guid, *p_lid;
+ uint64_t guid;
+ uint16_t lid, dlid;
+ int sl, mtu, rate;
+ int ret = 1, i;
+ uint8_t addr[ACM_MAX_ADDRESS];
+ uint8_t addr_type;
+
+ ibv_query_gid(ep->port->dev->verbs, ep->port->port_num, 0, &sgid);
+
+ /* Search for endpoint's SLID */
+ while (fgets(s, sizeof s, f)) {
+ if (s[0] == '#')
+ continue;
+ if (!(p = strtok_r(s, " \n", &ptr)))
+ continue; /* ignore blank lines */
+
+ if (strncmp(p, "Switch", sizeof("Switch") - 1) &&
+ strncmp(p, "Channel", sizeof("Channel") - 1) &&
+ strncmp(p, "Router", sizeof("Router") - 1))
+ continue;
+
+ if (!strncmp(p, "Channel", sizeof("Channel") - 1)) {
+ p = strtok_r(NULL, " ", &ptr); /* skip 'Adapter' */
+ if (!p)
+ continue;
+ }
+
+ p_guid = strtok_r(NULL, ",", &ptr);
+ if (!p_guid)
+ continue;
+
+ guid = (uint64_t) strtoull(p_guid, NULL, 16);
+ if (guid != ntohll(sgid.global.interface_id))
+ continue;
+
+ ptr = strstr(ptr, "base LID");
+ if (!ptr)
+ continue;
+ ptr += sizeof("base LID");
+ p_lid = strtok_r(NULL, ",", &ptr);
+ if (!p_lid)
+ continue;
+
+ lid = (uint16_t) strtoul(p_lid, NULL, 0);
+ if (lid != ep->port->lid)
+ continue;
+
+ ibv_query_port(ep->port->dev->verbs, ep->port->port_num, &attr);
+ ret = 0;
+ break;
+ }
+
+ while (fgets(s, sizeof s, f)) {
+ if (s[0] == '#')
+ continue;
+ if (!(p = strtok_r(s, " \n", &ptr)))
+ continue; /* ignore blank lines */
+
+ if (!strncmp(p, "Switch", sizeof("Switch") - 1) ||
+ !strncmp(p, "Channel", sizeof("Channel") - 1) ||
+ !strncmp(p, "Router", sizeof("Router") - 1))
+ break;
+
+ dlid = strtoul(p, NULL, 0);
+
+ p = strtok_r(NULL, ":", &ptr);
+ if (!p)
+ continue;
+ if (strcmp(p, "UNREACHABLE") == 0)
+ continue;
+ sl = atoi(p);
+
+ p = strtok_r(NULL, ":", &ptr);
+ if (!p)
+ continue;
+ mtu = atoi(p);
+
+ p = strtok_r(NULL, ":", &ptr);
+ if (!p)
+ continue;
+ rate = atoi(p);
+
+ if (!lid2guid[dlid]) {
+ acm_log(0, "ERROR - dlid %u not found in lid2guid table\n", dlid);
+ continue;
+ }
+
+ dgid.global.subnet_prefix = sgid.global.subnet_prefix;
+ dgid.global.interface_id = lid2guid[dlid];
+
+ for (i = 0; i < 2; i++) {
+ memset(addr, 0, ACM_MAX_ADDRESS);
+ if (i == 0) {
+ addr_type = ACM_ADDRESS_LID;
+ *((uint16_t *) addr) = htons(dlid);
+ } else {
+ addr_type = ACM_ADDRESS_GID;
+ memcpy(addr, &dgid, sizeof(dgid));
+ }
+ dest = acm_acquire_dest(ep, addr_type, addr);
+ if (!dest) {
+ acm_log(0, "ERROR - unable to create dest\n");
+ break;
+ }
+
+ dest->path.sgid = sgid;
+ dest->path.slid = htons(ep->port->lid);
+ dest->path.dgid = dgid;
+ dest->path.dlid = htons(dlid);
+ dest->path.reversible_numpath = IBV_PATH_RECORD_REVERSIBLE;
+ dest->path.pkey = htons(ep->pkey);
+ dest->path.mtu = (uint8_t) mtu;
+ dest->path.rate = (uint8_t) rate;
+ dest->path.qosclass_sl = htons((uint16_t) sl & 0xF);
+ if (dlid == ep->port->lid) {
+ dest->path.packetlifetime = 0;
+ dest->addr_timeout = (uint64_t)~0ULL;
+ dest->route_timeout = (uint64_t)~0ULL;
+ } else {
+ dest->path.packetlifetime = attr.subnet_timeout;
+ dest->addr_timeout = time_stamp_min() + (unsigned) addr_timeout;
+ dest->route_timeout = time_stamp_min() + (unsigned) route_timeout;
+ }
+ dest->remote_qpn = 1;
+ dest->state = ACM_READY;
+ acm_put_dest(dest);
+ acm_log(1, "added cached dest %s\n", dest->name);
+ }
+ }
+ return ret;
+}
+
+static int acm_parse_osm_fullv1(struct acm_ep *ep)
+{
+ FILE *f;
+ uint64_t *lid2guid;
+ int ret = 1;
+
+ if (!(f = fopen(route_data_file, "r"))) {
+ acm_log(0, "ERROR - couldn't open %s\n", route_data_file);
+ return ret;
+ }
+
+ lid2guid = calloc(IB_LID_MCAST_START, sizeof(*lid2guid));
+ if (!lid2guid) {
+ acm_log(0, "ERROR - no memory for path record parsing\n");
+ goto err;
+ }
+
+ acm_parse_osm_fullv1_lid2guid(f, lid2guid);
+ rewind(f);
+ ret = acm_parse_osm_fullv1_paths(f, lid2guid, ep);
+ free(lid2guid);
+err:
+ fclose(f);
+ return ret;
+}
+
static int acm_assign_ep_names(struct acm_ep *ep)
{
FILE *faddr;
return !index;
}
+static void acm_ep_preload(struct acm_ep *ep)
+{
+ switch (route_preload) {
+ case ACM_ROUTE_PRELOAD_OSM_FULL_V1:
+ if (acm_parse_osm_fullv1(ep))
+ acm_log(0, "ERROR - failed to preload EP\n");
+ break;
+ default:
+ break;
+ }
+}
+
static int acm_init_ep_loopback(struct acm_ep *ep)
{
struct acm_dest *dest;
lock_acquire(&port->lock);
DListInsertHead(&ep->entry, &port->ep_list);
lock_release(&port->lock);
+ acm_ep_preload(ep);
return;
err2:
min_mtu = acm_convert_mtu(atoi(value));
else if (!stricmp("min_rate", opt))
min_rate = acm_convert_rate(atoi(value));
+ else if (!stricmp("route_preload", opt))
+ route_preload = acm_convert_route_preload(value);
+ else if (!stricmp("route_data_file", opt))
+ strcpy(route_data_file, value);
}
fclose(f);
acm_log(0, "receive depth %d\n", recv_depth);
acm_log(0, "minimum mtu %d\n", min_mtu);
acm_log(0, "minimum rate %d\n", min_rate);
+ acm_log(0, "route preload %d\n", route_preload);
+ acm_log(0, "route data file %s\n", route_data_file);
}
static FILE *acm_open_log(void)
printf(" [-D] - run as a daemon (default)\n");
printf(" [-P] - run as a standard process\n");
printf(" [-A addr_file] - address configuration file\n");
- printf(" (default %s/%s\n", ACM_CONF_DIR, ACM_ADDR_FILE);
+ printf(" (default %s/%s)\n", ACM_CONF_DIR, ACM_ADDR_FILE);
printf(" [-O option_file] - option configuration file\n");
- printf(" (default %s/%s\n", ACM_CONF_DIR, ACM_OPTS_FILE);
+ printf(" (default %s/%s)\n", ACM_CONF_DIR, ACM_OPTS_FILE);
}
int CDECL_FUNC main(int argc, char **argv)
projects / ~shefty / ibacm.git / commitdiff