From bd556eb345ea9ea5a65eb350b31d05770e66bf08 Mon Sep 17 00:00:00 2001 From: Kaike Wan Date: Wed, 11 Jun 2014 10:45:09 -0700 Subject: [PATCH] ibacm/prov: Add a gid cache for each port The attempt is to eliminate frequent call of ibv_query_gid(). Signed-off-by: Kaike Wan Signed-off-by: Sean Hefty --- include/infiniband/acm_prov.h | 4 +- prov/acmp/src/acmp.c | 42 ++++++------ src/acm.c | 118 +++++++++++++++++++++------------- 3 files changed, 95 insertions(+), 69 deletions(-) diff --git a/include/infiniband/acm_prov.h b/include/infiniband/acm_prov.h index d4efb5a..dcfdf5f 100644 --- a/include/infiniband/acm_prov.h +++ b/include/infiniband/acm_prov.h @@ -94,8 +94,8 @@ extern void acm_format_name(int level, char *name, size_t name_size, uint8_t addr_type, const uint8_t *addr, size_t addr_size); extern int ib_any_gid(union ibv_gid *gid); -extern uint8_t acm_gid_index(struct ibv_context *verbs, int port_num, - int gid_cnt, union ibv_gid *gid); +extern uint8_t acm_gid_index(struct acm_port *port, union ibv_gid *gid); +extern int acm_get_gid(struct acm_port *port, int index, union ibv_gid *gid); extern uint64_t acm_path_comp_mask(struct ibv_path_record *path); extern int acm_resolve_response(uint64_t id, struct acm_msg *msg); diff --git a/prov/acmp/src/acmp.c b/prov/acmp/src/acmp.c index 7d59d1e..45ad03b 100644 --- a/prov/acmp/src/acmp.c +++ b/prov/acmp/src/acmp.c @@ -130,12 +130,10 @@ struct acmp_port { DLIST_ENTRY ep_list; lock_t lock; struct acmp_dest sa_dest; - union ibv_gid base_gid; enum ibv_port_state state; enum ibv_mtu mtu; enum ibv_rate rate; int subnet_timeout; - int gid_cnt; uint16_t default_pkey_ix; uint16_t lid; uint16_t lid_mask; @@ -640,8 +638,8 @@ acmp_record_mc_av(struct acmp_port *port, struct ib_mc_member_rec *mc_rec, dest->av.is_global = 1; dest->av.grh.dgid = mc_rec->mgid; dest->av.grh.flow_label = (sl_flow_hop >> 8) & 0xFFFFF; - dest->av.grh.sgid_index = acm_gid_index(port->dev->verbs, - port->port_num, port->gid_cnt, &mc_rec->port_gid); + dest->av.grh.sgid_index = acm_gid_index((struct acm_port *) port->port, + &mc_rec->port_gid); dest->av.grh.hop_limit = (uint8_t) sl_flow_hop; dest->av.grh.traffic_class = mc_rec->tclass; @@ -674,8 +672,13 @@ acmp_init_path_av(struct acmp_port *port, struct acmp_dest *dest) flow_hop = ntohl(dest->path.flowlabel_hoplimit); dest->av.is_global = 1; dest->av.grh.flow_label = (flow_hop >> 8) & 0xFFFFF; - dest->av.grh.sgid_index = acm_gid_index(port->dev->verbs, - port->port_num, port->gid_cnt, &dest->path.sgid); + lock_acquire(&port->lock); + if (port->port) + dest->av.grh.sgid_index = acm_gid_index( + (struct acm_port *) port->port, &dest->path.sgid); + else + dest->av.grh.sgid_index = 0; + lock_release(&port->lock); dest->av.grh.hop_limit = (uint8_t) flow_hop; dest->av.grh.traffic_class = dest->path.tclass; } @@ -1408,6 +1411,7 @@ static void acmp_join_group(struct acmp_ep *ep, union ibv_gid *port_gid, static void acmp_ep_join(struct acmp_ep *ep) { struct acmp_port *port; + union ibv_gid gid; port = ep->port; acm_log(1, "%s\n", ep->id_string); @@ -1420,11 +1424,12 @@ static void acmp_ep_join(struct acmp_ep *ep) } ep->mc_cnt = 0; ep->state = ACMP_INIT; - acmp_join_group(ep, &port->base_gid, 0, 0, 0, min_rate, min_mtu); + acm_get_gid((struct acm_port *)ep->port->port, 0, &gid); + acmp_join_group(ep, &gid, 0, 0, 0, min_rate, min_mtu); if ((route_prot == ACMP_ROUTE_PROT_ACM) && (port->rate != min_rate || port->mtu != min_mtu)) - acmp_join_group(ep, &port->base_gid, 0, 0, 0, port->rate, port->mtu); + acmp_join_group(ep, &gid, 0, 0, 0, port->rate, port->mtu); acm_log(1, "join for %s complete\n", ep->id_string); } @@ -2032,7 +2037,7 @@ static int acmp_parse_osm_fullv1_paths(FILE *f, uint64_t *lid2guid, struct acmp_ uint8_t addr[ACM_MAX_ADDRESS]; uint8_t addr_type; - ibv_query_gid(ep->port->dev->verbs, ep->port->port_num, 0, &sgid); + acm_get_gid((struct acm_port *)ep->port->port, 0, &sgid); /* Search for endpoint's SLID */ while (fgets(s, sizeof s, f)) { @@ -2317,9 +2322,7 @@ static int acmp_add_addr(const struct acm_address *addr, void *ep_context, return -1; } - ibv_query_gid(ep->port->dev->verbs, ep->port->port_num, - 0, &dest->path.sgid); - + acm_get_gid((struct acm_port *) ep->port->port, 0, &dest->path.sgid); dest->path.dgid = dest->path.sgid; dest->path.dlid = dest->path.slid = htons(ep->port->lid); dest->path.reversible_numpath = IBV_PATH_RECORD_REVERSIBLE; @@ -2550,7 +2553,6 @@ err0: static void acmp_port_up(struct acmp_port *port) { struct ibv_port_attr attr; - union ibv_gid gid; uint16_t pkey, sm_lid; int i, ret; @@ -2565,15 +2567,6 @@ static void acmp_port_up(struct acmp_port *port) port->rate = acm_get_rate(attr.active_width, attr.active_speed); if (attr.subnet_timeout >= 8) port->subnet_timeout = 1 << (attr.subnet_timeout - 8); - for (port->gid_cnt = 0;; port->gid_cnt++) { - ret = ibv_query_gid(port->dev->verbs, port->port_num, - port->gid_cnt, &gid); - if (ret || !gid.global.interface_id) - break; - - if (port->gid_cnt == 0) - port->base_gid = gid; - } port->lid = attr.lid; port->lid_mask = 0xffff - ((1 << attr.lmc) - 1); @@ -2644,8 +2637,10 @@ static int acmp_open_port(const struct acm_port *cport, void *dev_context, } port = &dev->port[cport->port_num - 1]; + lock_acquire(&port->lock); port->port = cport; port->state = IBV_PORT_DOWN; + lock_release(&port->lock); acmp_port_up(port); *port_context = port; return 0; @@ -2656,8 +2651,9 @@ static void acmp_close_port(void *port_context) struct acmp_port *port = port_context; acmp_port_down(port); + lock_acquire(&port->lock); port->port = NULL; - port->state = IBV_PORT_DOWN; + lock_release(&port->lock); } static void acmp_init_port(struct acmp_port *port, struct acmp_device *dev, diff --git a/src/acm.c b/src/acm.c index 146ca99..606998c 100644 --- a/src/acm.c +++ b/src/acm.c @@ -97,8 +97,8 @@ struct acmc_port { lock_t lock; DLIST_ENTRY ep_list; enum ibv_port_state state; - union ibv_gid base_gid; int gid_cnt; + union ibv_gid *gid_tbl; uint16_t lid; uint16_t lid_mask; }; @@ -312,20 +312,32 @@ acm_release_prov_context(struct acmc_prov_context *ctx) } } -uint8_t acm_gid_index(struct ibv_context *verbs, int port_num, - int gid_cnt, union ibv_gid *gid) +uint8_t acm_gid_index(struct acm_port *port, union ibv_gid *gid) { - union ibv_gid cmp_gid; uint8_t i; + struct acmc_port *cport; - for (i = 0; i < gid_cnt; i++) { - ibv_query_gid(verbs, port_num, i, &cmp_gid); - if (!memcmp(&cmp_gid, gid, sizeof cmp_gid)) + cport = container_of(port, struct acmc_port, port); + for (i = 0; i < cport->gid_cnt; i++) { + if (!memcmp(&cport->gid_tbl[i], gid, sizeof (*gid))) break; } return i; } +int acm_get_gid(struct acm_port *port, int index, union ibv_gid *gid) +{ + struct acmc_port *cport; + + cport = container_of(port, struct acmc_port, port); + if (index >= 0 && index < cport->gid_cnt) { + *gid = cport->gid_tbl[index]; + return 0; + } else { + return -1; + } +} + static void acm_mark_addr_invalid(struct acmc_ep *ep, struct acm_ep_addr_data *data) { @@ -589,13 +601,11 @@ static void acm_svr_accept(void) static int acm_is_path_from_port(struct acmc_port *port, struct ibv_path_record *path) { - union ibv_gid gid; uint8_t i; if (!ib_any_gid(&path->sgid)) { - return (acm_gid_index(port->dev->device.verbs, - port->port.port_num, port->gid_cnt, - &path->sgid) < port->gid_cnt); + return (acm_gid_index(&port->port, &path->sgid) < + port->gid_cnt); } if (path->slid) { @@ -606,15 +616,13 @@ acm_is_path_from_port(struct acmc_port *port, struct ibv_path_record *path) return 1; } - if (acm_gid_index(port->dev->device.verbs, port->port.port_num, - port->gid_cnt, &path->dgid) < port->gid_cnt) { + if (acm_gid_index(&port->port, &path->dgid) < port->gid_cnt) { return 1; } for (i = 0; i < port->gid_cnt; i++) { - ibv_query_gid(port->dev->device.verbs, port->port.port_num, - i, &gid); - if (gid.global.subnet_prefix == path->dgid.global.subnet_prefix) { + if (port->gid_tbl[i].global.subnet_prefix == + path->dgid.global.subnet_prefix) { return 1; } } @@ -1431,31 +1439,20 @@ acm_get_device_from_gid(union ibv_gid *sgid, uint8_t *port) { DLIST_ENTRY *dev_entry; struct acmc_device *dev; - struct ibv_device_attr dev_attr; - struct ibv_port_attr port_attr; - union ibv_gid gid; - int ret, i; + int i; for (dev_entry = dev_list.Next; dev_entry != &dev_list; dev_entry = dev_entry->Next) { dev = container_of(dev_entry, struct acmc_device, entry); - ret = ibv_query_device(dev->device.verbs, &dev_attr); - if (ret) - continue; + for (*port = 1; *port <= dev->port_cnt; (*port)++) { - for (*port = 1; *port <= dev_attr.phys_port_cnt; (*port)++) { - ret = ibv_query_port(dev->device.verbs, *port, &port_attr); - if (ret) - continue; - - for (i = 0; i < port_attr.gid_tbl_len; i++) { - ret = ibv_query_gid(dev->device.verbs, *port, i, &gid); - if (ret || !gid.global.interface_id) - break; + for (i = 0; i < dev->port[*port - 1].gid_cnt; i++) { - if (!memcmp(sgid->raw, gid.raw, sizeof gid)) + if (!memcmp(sgid->raw, + dev->port[*port - 1].gid_tbl[i].raw, + sizeof(*sgid))) return dev; } } @@ -1673,7 +1670,8 @@ static void acm_assign_provider(struct acmc_port *port) entry = entry->Next) { prov = container_of(entry, struct acmc_prov, entry); - if (prov->subnet_prefix == port->base_gid.global.subnet_prefix) { + if (prov->subnet_prefix == + port->gid_tbl[0].global.subnet_prefix) { acm_log(2, "Found provider %s for port %s/%d\n", prov->prov->name, port->port.dev->verbs->device->name, @@ -1693,10 +1691,45 @@ static void acm_assign_provider(struct acmc_port *port) } } +static void acm_port_get_gid_tbl(struct acmc_port *port) +{ + union ibv_gid gid; + int i, j, ret; + + for (i = 0;; i++) { + ret = ibv_query_gid(port->port.dev->verbs, port->port.port_num, + i, &gid); + if (ret || !gid.global.interface_id) + break; + } + + if (i > 0) { + port->gid_tbl = calloc(i, sizeof(union ibv_gid)); + if (!port->gid_tbl) { + acm_log(0, "Error: failed to allocate gid table\n"); + port->gid_cnt = 0; + return; + } + + for (j = 0; j < i; j++) { + ret = ibv_query_gid(port->port.dev->verbs, + port->port.port_num, j, + &port->gid_tbl[j]); + if (ret || !port->gid_tbl[j].global.interface_id) + break; + acm_log(2, "guid %d: 0x%llx %llx\n", j, + port->gid_tbl[j].global.subnet_prefix, + port->gid_tbl[j].global.interface_id); + } + port->gid_cnt = j; + } + acm_log(2, "port %d gid_cnt %d\n", port->port.port_num, + port->gid_cnt); +} + static void acm_port_up(struct acmc_port *port) { struct ibv_port_attr attr; - union ibv_gid gid; uint16_t pkey; int i, ret; struct acmc_prov_context *dev_ctx; @@ -1714,15 +1747,7 @@ static void acm_port_up(struct acmc_port *port) return; } - for (port->gid_cnt = 0;; port->gid_cnt++) { - ret = ibv_query_gid(port->dev->device.verbs, port->port.port_num, - port->gid_cnt, &gid); - if (ret || !gid.global.interface_id) - break; - if (port->gid_cnt == 0) - port->base_gid = gid; - } - + acm_port_get_gid_tbl(port); port->lid = attr.lid; port->lid_mask = 0xffff - ((1 << attr.lmc) - 1); port->sa_addr.lid = attr.sm_lid; @@ -1794,6 +1819,11 @@ static void acm_shutdown_port(struct acmc_port *port) } } port->prov = NULL; + if (port->gid_tbl) { + free(port->gid_tbl); + port->gid_tbl = NULL; + } + port->gid_cnt = 0; } static void acm_port_down(struct acmc_port *port) -- 2.46.0