From: Sean Hefty <sean.hefty@intel.com>
Date: Fri, 2 Oct 2009 19:48:47 +0000 (-0700)
Subject: libacm: open devices once
X-Git-Tag: 1.0.0~15
X-Git-Url: https://openfabrics.org/gitweb/?a=commitdiff_plain;h=cc1d35ec065de94cf267df1e468b00a34a9d6853;p=~shefty%2Fibacm.git

libacm: open devices once

Instead of opening IB devices on every call to ib_acm_convert_to_path,
open all devices during initialization and store the necessary data.
This optimizes multiple calls to ib_acm_convert_to_path to improve
connection setup times.

Signed-off-by: Sean Hefty <sean.hefty@intel.com>
---

diff --git a/src/libacm.c b/src/libacm.c
index 406f3a3..fe853ae 100644
--- a/src/libacm.c
+++ b/src/libacm.c
@@ -32,11 +32,113 @@
 #include <infiniband/acm.h>
 #include <stdio.h>
 
+struct acm_port
+{
+	uint8_t           port_num;
+	uint16_t          lid;
+	union ibv_gid     gid;
+	int               pkey_cnt;
+	uint16_t          pkey[4];
+};
+
+struct acm_device
+{
+	struct ibv_context *verbs;
+	uint64_t           guid;
+	int                port_cnt;
+	struct acm_port    *ports;
+};
+
 extern lock_t lock;
 static SOCKET sock = INVALID_SOCKET;
 static short server_port = 6125;
 static int ready;
 
+static struct acm_device *dev_array;
+static int dev_cnt;
+
+
+static int acm_init_port(struct acm_device *dev, int index)
+{
+	struct acm_port *port;
+	struct ibv_port_attr attr;
+	int ret;
+
+	port = &dev->ports[index];
+	port->port_num = index + 1;
+	ret = ibv_query_gid(dev->verbs, port->port_num, 0, &port->gid);
+	if (ret)
+		return -1;
+
+	ret = ibv_query_port(dev->verbs, port->port_num, &attr);
+	if (ret)
+		return -1;
+
+	port->lid = attr.lid;
+	for (port->pkey_cnt = 0; !ret && port->pkey_cnt < 4; port->pkey_cnt++) {
+		ret = ibv_query_pkey(dev->verbs, port->port_num,
+			port->pkey_cnt, &port->pkey[port->pkey_cnt]);
+	}
+
+	return port->pkey_cnt ? 0 : ret;
+}
+
+static int acm_open_devices(void)
+{
+	struct ibv_device **dev_list;
+	struct acm_device *dev;
+	struct ibv_device_attr attr;
+	int i, p, cnt, ret;
+
+	dev_list = ibv_get_device_list(&cnt);
+	if (!dev_list)
+		return -1;
+
+	dev_array = (struct acm_device *) zalloc(sizeof(struct acm_device) * cnt);
+	if (!dev_array)
+		goto err1;
+
+	for (i = 0; dev_list[i];) {
+		dev = &dev_array[i];
+
+		dev->guid = ibv_get_device_guid(dev_list[i]);
+		dev->verbs = ibv_open_device(dev_list[i]);
+		if (dev->verbs == NULL)
+			goto err2;
+
+		++i;
+		ret = ibv_query_device(dev->verbs, &attr);
+		if (ret)
+			goto err2;
+
+		dev->port_cnt = attr.phys_port_cnt;
+		dev->ports = zalloc(sizeof(struct acm_port) * dev->port_cnt);
+		if (!dev->ports)
+			goto err2;
+
+		for (p = 0; p < dev->port_cnt; p++) {
+			ret = acm_init_port(dev, p);
+			if (ret)
+				goto err2;
+		}
+	}
+
+	ibv_free_device_list(dev_list);
+	dev_cnt = cnt;
+	return 0;
+
+err2:
+	while (i--) {
+		ibv_close_device(dev_array[i].verbs);
+		if (dev_array[i].ports)
+			free(dev_array[i].ports);
+	}
+	free(dev_array);
+err1:
+	ibv_free_device_list(dev_list);
+	return -1;
+}
+
 static int acm_init(void)
 {
 	struct sockaddr_in addr;
@@ -60,6 +162,10 @@ static int acm_init(void)
 	if (ret)
 		goto err2;
 
+	ret = acm_open_devices();
+	if (ret)
+		goto err2;
+
 	ready = 1;
 	return 0;
 
@@ -221,25 +327,16 @@ int ib_acm_resolve_path(struct ib_path_record *path)
 	return acm_query_path(path, 0);
 }
 
-static struct ibv_context *acm_open_device(uint64_t guid)
+static struct acm_device *acm_get_device(uint64_t guid)
 {
-	struct ibv_device **dev_array;
-	struct ibv_context *verbs = NULL;
-	int i, cnt;
-
-	dev_array = ibv_get_device_list(&cnt);
-	if (!dev_array)
-		return NULL;
+	int i;
 
-	for (i = 0; i < cnt; i++) {
-		if (guid == ibv_get_device_guid(dev_array[i])) {
-			verbs = ibv_open_device(dev_array[i]);
-			break;
-		}
+	for (i = 0; i < dev_cnt; i++) {
+		if (dev_array[i].guid == guid)
+			return &dev_array[i];
 	}
 
-	ibv_free_device_list(dev_array);
-	return verbs;
+	return NULL;
 }
 
 LIB_EXPORT
@@ -247,43 +344,32 @@ int ib_acm_convert_to_path(struct ib_acm_dev_addr *dev_addr,
 	struct ibv_ah_attr *ah, struct ib_acm_resolve_data *data,
 	struct ib_path_record *path)
 {
-	struct ibv_context *verbs;
-	struct ibv_port_attr attr;
-	int ret;
+	struct acm_device *dev;
+	int p = dev_addr->port_num - 1;
+
+	dev = acm_get_device(dev_addr->guid);
+	if (!dev)
+		return -1;
 
-	verbs = acm_open_device(dev_addr->guid);
-	if (!verbs)
+	if (ah->grh.sgid_index || dev_addr->pkey_index > 4)
 		return -1;
 
 	if (ah->is_global) {
 		path->dgid = ah->grh.dgid;
-		ret = ibv_query_gid(verbs, dev_addr->port_num, ah->grh.sgid_index, &path->sgid);
-		if (ret)
-			goto out;
-
+		path->sgid = dev->ports[p].gid;
 		path->flowlabel_hoplimit =
 			htonl(ah->grh.flow_label << 8 | (uint32_t) ah->grh.hop_limit);
 		path->tclass = ah->grh.traffic_class;
 	}
 
 	path->dlid = htons(ah->dlid);
-	ret = ibv_query_port(verbs, dev_addr->port_num, &attr);
-	if (ret)
-		goto out;
-
-	path->slid = htons(attr.lid | ah->src_path_bits);
+	path->slid = htons(dev->ports[p].lid | ah->src_path_bits);
 	path->reversible_numpath = IB_PATH_RECORD_REVERSIBLE | 1;
-	ret = ibv_query_pkey(verbs, dev_addr->port_num, dev_addr->pkey_index, &path->pkey);
-	if (ret)
-		goto out;
-
-	path->pkey = htons(path->pkey);
+	path->pkey = htons(dev->ports[p].pkey[dev_addr->pkey_index]);
 	path->qosclass_sl = htons((uint16_t) ah->sl);
 	path->mtu = (2 << 6) | data->mtu;
 	path->rate = (2 << 6) | ah->static_rate;
 	path->packetlifetime = (2 << 6) | data->packet_lifetime;
 
-out:
-	ibv_close_device(verbs);
-	return ret;
+	return 0;
 }