libacm: open devices once

Instead of opening IB devices on every call to ib_acm_convert_to_path,
open all devices during initialization and store the necessary data.
This optimizes multiple calls to ib_acm_convert_to_path to improve
connection setup times.

Signed-off-by: Sean Hefty <sean.hefty@intel.com>

diff --git a/src/libacm.c b/src/libacm.c
index 406f3a3389a8afe1dfe85cbeb89464738a2fbe4b..fe853aecf72559019e25aba011a29942936228a9 100644 (file)
--- a/src/libacm.c
+++ b/src/libacm.c
@@ -32,11 +32,113 @@
 #include <infiniband/acm.h>\r
 #include <stdio.h>\r
 \r
+struct acm_port\r
+{\r
+       uint8_t           port_num;\r
+       uint16_t          lid;\r
+       union ibv_gid     gid;\r
+       int               pkey_cnt;\r
+       uint16_t          pkey[4];\r
+};\r
+\r
+struct acm_device\r
+{\r
+       struct ibv_context *verbs;\r
+       uint64_t           guid;\r
+       int                port_cnt;\r
+       struct acm_port    *ports;\r
+};\r
+\r
 extern lock_t lock;\r
 static SOCKET sock = INVALID_SOCKET;\r
 static short server_port = 6125;\r
 static int ready;\r
 \r
+static struct acm_device *dev_array;\r
+static int dev_cnt;\r
+\r
+\r
+static int acm_init_port(struct acm_device *dev, int index)\r
+{\r
+       struct acm_port *port;\r
+       struct ibv_port_attr attr;\r
+       int ret;\r
+\r
+       port = &dev->ports[index];\r
+       port->port_num = index + 1;\r
+       ret = ibv_query_gid(dev->verbs, port->port_num, 0, &port->gid);\r
+       if (ret)\r
+               return -1;\r
+\r
+       ret = ibv_query_port(dev->verbs, port->port_num, &attr);\r
+       if (ret)\r
+               return -1;\r
+\r
+       port->lid = attr.lid;\r
+       for (port->pkey_cnt = 0; !ret && port->pkey_cnt < 4; port->pkey_cnt++) {\r
+               ret = ibv_query_pkey(dev->verbs, port->port_num,\r
+                       port->pkey_cnt, &port->pkey[port->pkey_cnt]);\r
+       }\r
+\r
+       return port->pkey_cnt ? 0 : ret;\r
+}\r
+\r
+static int acm_open_devices(void)\r
+{\r
+       struct ibv_device **dev_list;\r
+       struct acm_device *dev;\r
+       struct ibv_device_attr attr;\r
+       int i, p, cnt, ret;\r
+\r
+       dev_list = ibv_get_device_list(&cnt);\r
+       if (!dev_list)\r
+               return -1;\r
+\r
+       dev_array = (struct acm_device *) zalloc(sizeof(struct acm_device) * cnt);\r
+       if (!dev_array)\r
+               goto err1;\r
+\r
+       for (i = 0; dev_list[i];) {\r
+               dev = &dev_array[i];\r
+\r
+               dev->guid = ibv_get_device_guid(dev_list[i]);\r
+               dev->verbs = ibv_open_device(dev_list[i]);\r
+               if (dev->verbs == NULL)\r
+                       goto err2;\r
+\r
+               ++i;\r
+               ret = ibv_query_device(dev->verbs, &attr);\r
+               if (ret)\r
+                       goto err2;\r
+\r
+               dev->port_cnt = attr.phys_port_cnt;\r
+               dev->ports = zalloc(sizeof(struct acm_port) * dev->port_cnt);\r
+               if (!dev->ports)\r
+                       goto err2;\r
+\r
+               for (p = 0; p < dev->port_cnt; p++) {\r
+                       ret = acm_init_port(dev, p);\r
+                       if (ret)\r
+                               goto err2;\r
+               }\r
+       }\r
+\r
+       ibv_free_device_list(dev_list);\r
+       dev_cnt = cnt;\r
+       return 0;\r
+\r
+err2:\r
+       while (i--) {\r
+               ibv_close_device(dev_array[i].verbs);\r
+               if (dev_array[i].ports)\r
+                       free(dev_array[i].ports);\r
+       }\r
+       free(dev_array);\r
+err1:\r
+       ibv_free_device_list(dev_list);\r
+       return -1;\r
+}\r
+\r
 static int acm_init(void)\r
 {\r
        struct sockaddr_in addr;\r
@@ -60,6 +162,10 @@ static int acm_init(void)
        if (ret)\r
                goto err2;\r
 \r
+       ret = acm_open_devices();\r
+       if (ret)\r
+               goto err2;\r
+\r
        ready = 1;\r
        return 0;\r
 \r
@@ -221,25 +327,16 @@ int ib_acm_resolve_path(struct ib_path_record *path)
        return acm_query_path(path, 0);\r
 }\r
 \r
-static struct ibv_context *acm_open_device(uint64_t guid)\r
+static struct acm_device *acm_get_device(uint64_t guid)\r
 {\r
-       struct ibv_device **dev_array;\r
-       struct ibv_context *verbs = NULL;\r
-       int i, cnt;\r
-\r
-       dev_array = ibv_get_device_list(&cnt);\r
-       if (!dev_array)\r
-               return NULL;\r
+       int i;\r
 \r
-       for (i = 0; i < cnt; i++) {\r
-               if (guid == ibv_get_device_guid(dev_array[i])) {\r
-                       verbs = ibv_open_device(dev_array[i]);\r
-                       break;\r
-               }\r
+       for (i = 0; i < dev_cnt; i++) {\r
+               if (dev_array[i].guid == guid)\r
+                       return &dev_array[i];\r
        }\r
 \r
-       ibv_free_device_list(dev_array);\r
-       return verbs;\r
+       return NULL;\r
 }\r
 \r
 LIB_EXPORT\r
@@ -247,43 +344,32 @@ int ib_acm_convert_to_path(struct ib_acm_dev_addr *dev_addr,
        struct ibv_ah_attr *ah, struct ib_acm_resolve_data *data,\r
        struct ib_path_record *path)\r
 {\r
-       struct ibv_context *verbs;\r
-       struct ibv_port_attr attr;\r
-       int ret;\r
+       struct acm_device *dev;\r
+       int p = dev_addr->port_num - 1;\r
+\r
+       dev = acm_get_device(dev_addr->guid);\r
+       if (!dev)\r
+               return -1;\r
 \r
-       verbs = acm_open_device(dev_addr->guid);\r
-       if (!verbs)\r
+       if (ah->grh.sgid_index || dev_addr->pkey_index > 4)\r
                return -1;\r
 \r
        if (ah->is_global) {\r
                path->dgid = ah->grh.dgid;\r
-               ret = ibv_query_gid(verbs, dev_addr->port_num, ah->grh.sgid_index, &path->sgid);\r
-               if (ret)\r
-                       goto out;\r
-\r
+               path->sgid = dev->ports[p].gid;\r
                path->flowlabel_hoplimit =\r
                        htonl(ah->grh.flow_label << 8 | (uint32_t) ah->grh.hop_limit);\r
                path->tclass = ah->grh.traffic_class;\r
        }\r
 \r
        path->dlid = htons(ah->dlid);\r
-       ret = ibv_query_port(verbs, dev_addr->port_num, &attr);\r
-       if (ret)\r
-               goto out;\r
-\r
-       path->slid = htons(attr.lid | ah->src_path_bits);\r
+       path->slid = htons(dev->ports[p].lid | ah->src_path_bits);\r
        path->reversible_numpath = IB_PATH_RECORD_REVERSIBLE | 1;\r
-       ret = ibv_query_pkey(verbs, dev_addr->port_num, dev_addr->pkey_index, &path->pkey);\r
-       if (ret)\r
-               goto out;\r
-\r
-       path->pkey = htons(path->pkey);\r
+       path->pkey = htons(dev->ports[p].pkey[dev_addr->pkey_index]);\r
        path->qosclass_sl = htons((uint16_t) ah->sl);\r
        path->mtu = (2 << 6) | data->mtu;\r
        path->rate = (2 << 6) | ah->static_rate;\r
        path->packetlifetime = (2 << 6) | data->packet_lifetime;\r
 \r
-out:\r
-       ibv_close_device(verbs);\r
-       return ret;\r
+       return 0;\r
 }\r