ibacm/prov: Add a gid cache for each port

The attempt is to eliminate frequent call of ibv_query_gid().

Signed-off-by: Kaike Wan <kaike.wan@intel.com>
Signed-off-by: Sean Hefty <sean.hefty@intel.com>

diff --git a/include/infiniband/acm_prov.h b/include/infiniband/acm_prov.h
index d4efb5a2bd8ee4c396532121fc3126c7d8cf9c73..dcfdf5fce7373a7d57bcbe3fe54d6fe5d81935e2 100644 (file)
--- a/include/infiniband/acm_prov.h
+++ b/include/infiniband/acm_prov.h
@@ -94,8 +94,8 @@ extern void acm_format_name(int level, char *name, size_t name_size,
        uint8_t addr_type, const uint8_t *addr, size_t addr_size);
 
 extern int ib_any_gid(union ibv_gid *gid);
-extern uint8_t acm_gid_index(struct ibv_context *verbs, int port_num, 
-       int gid_cnt, union ibv_gid *gid);
+extern uint8_t acm_gid_index(struct acm_port *port, union ibv_gid *gid);
+extern int acm_get_gid(struct acm_port *port, int index, union ibv_gid *gid);
 extern uint64_t acm_path_comp_mask(struct ibv_path_record *path);
 
 extern int acm_resolve_response(uint64_t id, struct acm_msg *msg);

diff --git a/prov/acmp/src/acmp.c b/prov/acmp/src/acmp.c
index 7d59d1e8f122777a9cf7c792ef165ac36439912d..45ad03b9caac11d649d57287346e01f4f561e208 100644 (file)
--- a/prov/acmp/src/acmp.c
+++ b/prov/acmp/src/acmp.c
@@ -130,12 +130,10 @@ struct acmp_port {
        DLIST_ENTRY         ep_list;
        lock_t              lock;
        struct acmp_dest    sa_dest;
-       union ibv_gid       base_gid;
        enum ibv_port_state state;
        enum ibv_mtu        mtu;
        enum ibv_rate       rate;
        int                 subnet_timeout;
-       int                 gid_cnt;
        uint16_t            default_pkey_ix;
        uint16_t            lid;
        uint16_t            lid_mask;
@@ -640,8 +638,8 @@ acmp_record_mc_av(struct acmp_port *port, struct ib_mc_member_rec *mc_rec,
        dest->av.is_global = 1;
        dest->av.grh.dgid = mc_rec->mgid;
        dest->av.grh.flow_label = (sl_flow_hop >> 8) & 0xFFFFF;
-       dest->av.grh.sgid_index = acm_gid_index(port->dev->verbs, 
-               port->port_num, port->gid_cnt, &mc_rec->port_gid);
+       dest->av.grh.sgid_index = acm_gid_index((struct acm_port *) port->port,
+                                               &mc_rec->port_gid);
        dest->av.grh.hop_limit = (uint8_t) sl_flow_hop;
        dest->av.grh.traffic_class = mc_rec->tclass;
 
@@ -674,8 +672,13 @@ acmp_init_path_av(struct acmp_port *port, struct acmp_dest *dest)
        flow_hop = ntohl(dest->path.flowlabel_hoplimit);
        dest->av.is_global = 1;
        dest->av.grh.flow_label = (flow_hop >> 8) & 0xFFFFF;
-       dest->av.grh.sgid_index = acm_gid_index(port->dev->verbs,
-               port->port_num, port->gid_cnt, &dest->path.sgid);
+       lock_acquire(&port->lock);
+       if (port->port) 
+               dest->av.grh.sgid_index = acm_gid_index(
+                  (struct acm_port *) port->port, &dest->path.sgid);
+       else
+               dest->av.grh.sgid_index = 0;
+       lock_release(&port->lock);
        dest->av.grh.hop_limit = (uint8_t) flow_hop;
        dest->av.grh.traffic_class = dest->path.tclass;
 }
@@ -1408,6 +1411,7 @@ static void acmp_join_group(struct acmp_ep *ep, union ibv_gid *port_gid,
 static void acmp_ep_join(struct acmp_ep *ep)
 {
        struct acmp_port *port;
+       union ibv_gid gid;
 
        port = ep->port;
        acm_log(1, "%s\n", ep->id_string);
@@ -1420,11 +1424,12 @@ static void acmp_ep_join(struct acmp_ep *ep)
        }
        ep->mc_cnt = 0;
        ep->state = ACMP_INIT;
-       acmp_join_group(ep, &port->base_gid, 0, 0, 0, min_rate, min_mtu);
+       acm_get_gid((struct acm_port *)ep->port->port, 0, &gid);
+       acmp_join_group(ep, &gid, 0, 0, 0, min_rate, min_mtu);
 
        if ((route_prot == ACMP_ROUTE_PROT_ACM) &&
            (port->rate != min_rate || port->mtu != min_mtu))
-               acmp_join_group(ep, &port->base_gid, 0, 0, 0, port->rate, port->mtu);
+               acmp_join_group(ep, &gid, 0, 0, 0, port->rate, port->mtu);
 
        acm_log(1, "join for %s complete\n", ep->id_string);
 }
@@ -2032,7 +2037,7 @@ static int acmp_parse_osm_fullv1_paths(FILE *f, uint64_t *lid2guid, struct acmp_
        uint8_t addr[ACM_MAX_ADDRESS];
        uint8_t addr_type;
 
-       ibv_query_gid(ep->port->dev->verbs, ep->port->port_num, 0, &sgid);
+       acm_get_gid((struct acm_port *)ep->port->port, 0, &sgid);
 
        /* Search for endpoint's SLID */
        while (fgets(s, sizeof s, f)) {
@@ -2317,9 +2322,7 @@ static int acmp_add_addr(const struct acm_address *addr, void *ep_context,
                return -1;
        }
 
-       ibv_query_gid(ep->port->dev->verbs, ep->port->port_num,
-                     0, &dest->path.sgid);
-
+       acm_get_gid((struct acm_port *) ep->port->port, 0, &dest->path.sgid);
        dest->path.dgid = dest->path.sgid;
        dest->path.dlid = dest->path.slid = htons(ep->port->lid);
        dest->path.reversible_numpath = IBV_PATH_RECORD_REVERSIBLE;
@@ -2550,7 +2553,6 @@ err0:
 static void acmp_port_up(struct acmp_port *port)
 {
        struct ibv_port_attr attr;
-       union ibv_gid gid;
        uint16_t pkey, sm_lid;
        int i, ret;
 
@@ -2565,15 +2567,6 @@ static void acmp_port_up(struct acmp_port *port)
        port->rate = acm_get_rate(attr.active_width, attr.active_speed);
        if (attr.subnet_timeout >= 8)
                port->subnet_timeout = 1 << (attr.subnet_timeout - 8);
-       for (port->gid_cnt = 0;; port->gid_cnt++) {
-               ret = ibv_query_gid(port->dev->verbs, port->port_num,
-                                   port->gid_cnt, &gid);
-               if (ret || !gid.global.interface_id)
-                       break;
-
-               if (port->gid_cnt == 0)
-                       port->base_gid = gid;
-       }
 
        port->lid = attr.lid;
        port->lid_mask = 0xffff - ((1 << attr.lmc) - 1);
@@ -2644,8 +2637,10 @@ static int acmp_open_port(const struct acm_port *cport, void *dev_context,
        }
 
        port = &dev->port[cport->port_num - 1];
+       lock_acquire(&port->lock);
        port->port = cport;
        port->state = IBV_PORT_DOWN;
+       lock_release(&port->lock);
        acmp_port_up(port);
        *port_context = port;
        return 0;
@@ -2656,8 +2651,9 @@ static void acmp_close_port(void *port_context)
        struct acmp_port *port = port_context;
 
        acmp_port_down(port);
+       lock_acquire(&port->lock);
        port->port = NULL;
-       port->state = IBV_PORT_DOWN;
+       lock_release(&port->lock);
 }
 
 static void acmp_init_port(struct acmp_port *port, struct acmp_device *dev, 

diff --git a/src/acm.c b/src/acm.c
index 146ca992b28f6c891ed73cce1b348798ce570409..606998c1be3e13e54132c615e061719e5f2ba755 100644 (file)
--- a/src/acm.c
+++ b/src/acm.c
@@ -97,8 +97,8 @@ struct acmc_port {
        lock_t              lock;
        DLIST_ENTRY         ep_list;
        enum ibv_port_state state;
-       union ibv_gid       base_gid;
        int                 gid_cnt;
+       union ibv_gid       *gid_tbl;
        uint16_t            lid;
        uint16_t            lid_mask;
 };
@@ -312,20 +312,32 @@ acm_release_prov_context(struct acmc_prov_context *ctx)
        }
 }
 
-uint8_t acm_gid_index(struct ibv_context *verbs, int port_num, 
-       int gid_cnt, union ibv_gid *gid)
+uint8_t acm_gid_index(struct acm_port *port, union ibv_gid *gid)
 {
-       union ibv_gid cmp_gid;
        uint8_t i;
+       struct acmc_port *cport;
 
-       for (i = 0; i < gid_cnt; i++) {
-               ibv_query_gid(verbs, port_num, i, &cmp_gid);
-               if (!memcmp(&cmp_gid, gid, sizeof cmp_gid))
+       cport = container_of(port, struct acmc_port, port);
+       for (i = 0; i < cport->gid_cnt; i++) {
+               if (!memcmp(&cport->gid_tbl[i], gid, sizeof (*gid)))
                        break;
        }
        return i;
 }
 
+int acm_get_gid(struct acm_port *port, int index, union ibv_gid *gid)
+{
+       struct acmc_port *cport;
+
+       cport = container_of(port, struct acmc_port, port);
+       if (index >= 0 && index < cport->gid_cnt) {
+               *gid = cport->gid_tbl[index];
+               return 0;
+       } else {
+               return -1;
+       }
+}
+
 static void acm_mark_addr_invalid(struct acmc_ep *ep,
                                  struct acm_ep_addr_data *data)
 {
@@ -589,13 +601,11 @@ static void acm_svr_accept(void)
 static int
 acm_is_path_from_port(struct acmc_port *port, struct ibv_path_record *path)
 {
-       union ibv_gid gid;
        uint8_t i;
 
        if (!ib_any_gid(&path->sgid)) {
-               return (acm_gid_index(port->dev->device.verbs, 
-                                     port->port.port_num, port->gid_cnt,
-                                     &path->sgid) < port->gid_cnt);
+               return (acm_gid_index(&port->port, &path->sgid) <
+                       port->gid_cnt);
        }
 
        if (path->slid) {
@@ -606,15 +616,13 @@ acm_is_path_from_port(struct acmc_port *port, struct ibv_path_record *path)
                return 1;
        }
 
-       if (acm_gid_index(port->dev->device.verbs, port->port.port_num, 
-                         port->gid_cnt, &path->dgid) < port->gid_cnt) {
+       if (acm_gid_index(&port->port, &path->dgid) < port->gid_cnt) {
                return 1;
        }
 
        for (i = 0; i < port->gid_cnt; i++) {
-               ibv_query_gid(port->dev->device.verbs, port->port.port_num, 
-                             i, &gid);
-               if (gid.global.subnet_prefix == path->dgid.global.subnet_prefix) {
+               if (port->gid_tbl[i].global.subnet_prefix == 
+                   path->dgid.global.subnet_prefix) {
                        return 1;
                }
        }
@@ -1431,31 +1439,20 @@ acm_get_device_from_gid(union ibv_gid *sgid, uint8_t *port)
 {
        DLIST_ENTRY *dev_entry;
        struct acmc_device *dev;
-       struct ibv_device_attr dev_attr;
-       struct ibv_port_attr port_attr;
-       union ibv_gid gid;
-       int ret, i;
+       int i;
 
        for (dev_entry = dev_list.Next; dev_entry != &dev_list;
                 dev_entry = dev_entry->Next) {
 
                dev = container_of(dev_entry, struct acmc_device, entry);
 
-               ret = ibv_query_device(dev->device.verbs, &dev_attr);
-               if (ret)
-                       continue;
+               for (*port = 1; *port <= dev->port_cnt; (*port)++) {
 
-               for (*port = 1; *port <= dev_attr.phys_port_cnt; (*port)++) {
-                       ret = ibv_query_port(dev->device.verbs, *port, &port_attr);
-                       if (ret)
-                               continue;
-
-                       for (i = 0; i < port_attr.gid_tbl_len; i++) {
-                               ret = ibv_query_gid(dev->device.verbs, *port, i, &gid);
-                               if (ret || !gid.global.interface_id)
-                                       break;
+                       for (i = 0; i < dev->port[*port - 1].gid_cnt; i++) {
 
-                               if (!memcmp(sgid->raw, gid.raw, sizeof gid))
+                               if (!memcmp(sgid->raw, 
+                                           dev->port[*port - 1].gid_tbl[i].raw, 
+                                           sizeof(*sgid)))
                                        return dev;
                        }
                }
@@ -1673,7 +1670,8 @@ static void acm_assign_provider(struct acmc_port *port)
             entry = entry->Next) {
                prov = container_of(entry, struct acmc_prov, entry);
 
-               if (prov->subnet_prefix == port->base_gid.global.subnet_prefix) {
+               if (prov->subnet_prefix == 
+                   port->gid_tbl[0].global.subnet_prefix) {
                        acm_log(2, "Found provider %s for port %s/%d\n",
                                prov->prov->name, 
                                port->port.dev->verbs->device->name, 
@@ -1693,10 +1691,45 @@ static void acm_assign_provider(struct acmc_port *port)
        } 
 }
 
+static void acm_port_get_gid_tbl(struct acmc_port *port)
+{
+       union ibv_gid gid;
+       int i, j, ret;
+
+       for (i = 0;; i++) {
+               ret = ibv_query_gid(port->port.dev->verbs, port->port.port_num, 
+                                   i, &gid);
+               if (ret || !gid.global.interface_id)
+                       break;
+       }
+
+       if (i > 0) {
+               port->gid_tbl = calloc(i, sizeof(union ibv_gid));
+               if (!port->gid_tbl) {
+                       acm_log(0, "Error: failed to allocate gid table\n");
+                       port->gid_cnt = 0;
+                       return;
+               }
+
+               for (j = 0; j < i; j++) {
+                       ret = ibv_query_gid(port->port.dev->verbs, 
+                                           port->port.port_num, j, 
+                                           &port->gid_tbl[j]);
+                       if (ret || !port->gid_tbl[j].global.interface_id)
+                               break;
+                       acm_log(2, "guid %d: 0x%llx %llx\n", j,
+                               port->gid_tbl[j].global.subnet_prefix, 
+                               port->gid_tbl[j].global.interface_id);
+               }
+               port->gid_cnt = j;
+       }
+       acm_log(2, "port %d gid_cnt %d\n", port->port.port_num, 
+               port->gid_cnt);
+}
+
 static void acm_port_up(struct acmc_port *port)
 {
        struct ibv_port_attr attr;
-       union ibv_gid gid;
        uint16_t pkey;
        int i, ret;
        struct acmc_prov_context *dev_ctx;
@@ -1714,15 +1747,7 @@ static void acm_port_up(struct acmc_port *port)
                return;
        }
 
-       for (port->gid_cnt = 0;; port->gid_cnt++) {
-               ret = ibv_query_gid(port->dev->device.verbs, port->port.port_num, 
-                                   port->gid_cnt, &gid);
-               if (ret || !gid.global.interface_id)
-                       break;
-               if (port->gid_cnt == 0)
-                       port->base_gid = gid;
-       }
-
+       acm_port_get_gid_tbl(port);
        port->lid = attr.lid;
        port->lid_mask = 0xffff - ((1 << attr.lmc) - 1);
        port->sa_addr.lid = attr.sm_lid;
@@ -1794,6 +1819,11 @@ static void acm_shutdown_port(struct acmc_port *port)
                }
        }
        port->prov = NULL;
+       if (port->gid_tbl) {
+               free(port->gid_tbl);
+               port->gid_tbl = NULL;
+       }
+       port->gid_cnt = 0;
 }
 
 static void acm_port_down(struct acmc_port *port)