From: Sean Hefty Date: Fri, 14 Jan 2011 16:47:32 +0000 (-0800) Subject: commit X-Git-Url: https://openfabrics.org/gitweb/?a=commitdiff_plain;h=13283e9884cea31f3ffb88bff68eb72f3b916d4e;p=~shefty%2Fibacm.git commit --- diff --git a/meta b/meta index 8971810..9fc44c1 100644 --- a/meta +++ b/meta @@ -1,8 +1,7 @@ Version: 1 -Previous: 68a998702961a4ae410846c4ed277bb9520abe9c +Previous: be3c10bb5263d34dfc4df62103aed6d3544a48b9 Head: 3b1f7191c890140c64eb9972876b4e56955a65ca Applied: - lid: 0f9837af4f64eeeae8a4c168c8ccf47186c735d4 acme_gid: 69a4139e95401a3100a2c363675fdd6d96a60d54 range: 1a80d1d2efa888ad294bac3e681f744555d4e816 format_lid: 3d001d786a9894e9a7e86932685e7c43568484b5 diff --git a/patches/lid b/patches/lid deleted file mode 100644 index 8c76307..0000000 --- a/patches/lid +++ /dev/null @@ -1,434 +0,0 @@ -Bottom: b2af32a606dc3c3e64f150250320dc7d52426926 -Top: 9cfb1ddbdd89bfe7a45eb80019ef27af2267dd2e -Author: Sean Hefty -Date: 2010-12-13 10:35:40 -0800 - -ibacm: add support for path lookup by LID/GID - -Support path record lookup based on DLID or DGID. This allows -ibacm to be used purely for path record caching, with address -resolution being done by a user. This is needed to support -clients that exchange LID/GID information out of band before -connecting, or that make use of ipoib for address resolution. - -Signed-off-by: Sean Hefty - - ---- - -diff --git a/src/acm.c b/src/acm.c -index e0f0b94..b4e0632 100644 ---- a/src/acm.c -+++ b/src/acm.c -@@ -268,8 +268,13 @@ acm_format_name(int level, char *name, size_t name_size, - break; - case ACM_EP_INFO_PATH: - path = (struct ibv_path_record *) addr; -- sprintf(name, "SLID(%d) DLID(%d)", -- ntohs(path->slid), ntohs(path->dlid)); -+ if (path->dlid) { -+ sprintf(name, "SLID(%d) DLID(%d)", -+ ntohs(path->slid), ntohs(path->dlid)); -+ } else { -+ acm_format_name(level, name, name_size, ACM_ADDRESS_GID, -+ path->dgid.raw, sizeof path->dgid); -+ } - break; - case ACM_ADDRESS_LID: - sprintf(name, "LID(%d)", *((uint16_t *) addr)); -@@ -280,6 +285,11 @@ acm_format_name(int level, char *name, size_t name_size, - } - } - -+static int ib_any_gid(union ibv_gid *gid) -+{ -+ return ((gid->global.subnet_prefix | gid->global.interface_id) == 0); -+} -+ - static int acm_compare_dest(const void *dest1, const void *dest2) - { - return memcmp(dest1, dest2, ACM_MAX_ADDRESS); -@@ -726,9 +736,23 @@ static int acm_addr_index(struct acm_ep *ep, uint8_t *addr, uint8_t addr_type) - static uint8_t - acm_record_acm_route(struct acm_ep *ep, struct acm_dest *dest) - { -+ int i; - uint8_t status; - - acm_log(2, "\n"); -+ for (i = 0; i < MAX_EP_MC; i++) { -+ if (!memcmp(&dest->mgid, &ep->mc_dest[i].mgid, sizeof dest->mgid)) -+ break; -+ } -+ if (i == MAX_EP_MC) { -+ acm_log(0, "ERROR - cannot match mgid\n"); -+ return ACM_STATUS_EINVAL; -+ } -+ -+ dest->path = ep->mc_dest[i].path; -+ dest->path.dgid = dest->av.grh.dgid; -+ dest->path.dlid = htons(dest->av.dlid); -+ - dest->ah = ibv_create_ah(ep->port->dev->pd, &dest->av); - if (!dest->ah) { - acm_log(0, "ERROR - failed to create ah\n"); -@@ -753,6 +777,59 @@ static void acm_init_path_query(struct ib_sa_mad *mad) - mad->attr_id = IB_SA_ATTR_PATH_REC; - } - -+static uint64_t acm_path_comp_mask(struct ibv_path_record *path) -+{ -+ uint32_t fl_hop; -+ uint16_t qos_sl; -+ uint64_t comp_mask = 0; -+ -+ acm_log(2, "\n"); -+ if (path->service_id) -+ comp_mask |= IB_COMP_MASK_PR_SERVICE_ID; -+ if (!ib_any_gid(&path->dgid)) -+ comp_mask |= IB_COMP_MASK_PR_DGID; -+ if (!ib_any_gid(&path->sgid)) -+ comp_mask |= IB_COMP_MASK_PR_SGID; -+ if (path->dlid) -+ comp_mask |= IB_COMP_MASK_PR_DLID; -+ if (path->slid) -+ comp_mask |= IB_COMP_MASK_PR_SLID; -+ -+ fl_hop = ntohl(path->flowlabel_hoplimit); -+ if (fl_hop >> 8) -+ comp_mask |= IB_COMP_MASK_PR_FLOW_LABEL; -+ if (fl_hop & 0xFF) -+ comp_mask |= IB_COMP_MASK_PR_HOP_LIMIT; -+ -+ if (path->tclass) -+ comp_mask |= IB_COMP_MASK_PR_TCLASS; -+ if (path->reversible_numpath & 0x80) -+ comp_mask |= IB_COMP_MASK_PR_REVERSIBLE; -+ if (path->pkey) -+ comp_mask |= IB_COMP_MASK_PR_PKEY; -+ -+ qos_sl = ntohs(path->qosclass_sl); -+ if (qos_sl >> 4) -+ comp_mask |= IB_COMP_MASK_PR_QOS_CLASS; -+ if (qos_sl & 0xF) -+ comp_mask |= IB_COMP_MASK_PR_SL; -+ -+ if (path->mtu & 0xC0) -+ comp_mask |= IB_COMP_MASK_PR_MTU_SELECTOR; -+ if (path->mtu & 0x3F) -+ comp_mask |= IB_COMP_MASK_PR_MTU; -+ if (path->rate & 0xC0) -+ comp_mask |= IB_COMP_MASK_PR_RATE_SELECTOR; -+ if (path->rate & 0x3F) -+ comp_mask |= IB_COMP_MASK_PR_RATE; -+ if (path->packetlifetime & 0xC0) -+ comp_mask |= IB_COMP_MASK_PR_PACKET_LIFETIME_SELECTOR; -+ if (path->packetlifetime & 0x3F) -+ comp_mask |= IB_COMP_MASK_PR_PACKET_LIFETIME; -+ -+ return comp_mask; -+} -+ - /* Caller must hold dest lock */ - static uint8_t acm_resolve_path(struct acm_ep *ep, struct acm_dest *dest, - void (*resp_handler)(struct acm_send_msg *req, -@@ -775,8 +852,7 @@ static uint8_t acm_resolve_path(struct acm_ep *ep, struct acm_dest *dest, - acm_init_path_query(mad); - - memcpy(mad->data, &dest->path, sizeof(dest->path)); -- mad->comp_mask = IB_COMP_MASK_PR_DGID | IB_COMP_MASK_PR_SGID | -- IB_COMP_MASK_PR_TCLASS | IB_COMP_MASK_PR_PKEY; -+ mad->comp_mask = acm_path_comp_mask(&dest->path); - - dest->state = ACM_QUERY_ROUTE; - acm_post_send(&ep->sa_queue, msg); -@@ -804,16 +880,29 @@ acm_record_acm_addr(struct acm_ep *ep, struct acm_dest *dest, struct ibv_wc *wc, - dest->av.grh.dgid = ((struct ibv_grh *) (uintptr_t) wc->wr_id)->sgid; - - dest->mgid = ep->mc_dest[index].mgid; -- -- dest->path = ep->mc_dest[index].path; -+ dest->path.sgid = ep->mc_dest[index].path.sgid; - dest->path.dgid = dest->av.grh.dgid; -- dest->path.dlid = htons(dest->av.dlid); -+ dest->path.tclass = ep->mc_dest[index].path.tclass; -+ dest->path.pkey = ep->mc_dest[index].path.pkey; - dest->remote_qpn = wc->src_qp; - - dest->state = ACM_ADDR_RESOLVED; - return ACM_STATUS_SUCCESS; - } - -+static void -+acm_record_path_addr(struct acm_ep *ep, struct acm_dest *dest, -+ struct ibv_path_record *path) -+{ -+ acm_log(2, "%s\n", dest->name); -+ dest->path.pkey = htons(ep->pkey); -+ dest->path.sgid = path->sgid; -+ dest->path.dgid = path->dgid; -+ dest->path.slid = path->slid; -+ dest->path.dlid = path->dlid; -+ dest->state = ACM_ADDR_RESOLVED; -+} -+ - static uint8_t acm_validate_addr_req(struct acm_mad *mad) - { - struct acm_resolve_rec *rec; -@@ -959,10 +1048,12 @@ acm_dest_sa_resp(struct acm_send_msg *msg, struct ibv_wc *wc, struct acm_mad *ma - if (!status) { - memcpy(&dest->path, sa_mad->data, sizeof(dest->path)); - acm_init_path_av(msg->ep->port, dest); -- dest->ah = ibv_create_ah(msg->ep->port->dev->pd, &dest->av); -- if (!dest->ah) { -- acm_log(0, "ERROR - failed to create ah\n"); -- status = ACM_STATUS_ENOMEM; -+ if (dest->remote_qpn) { -+ dest->ah = ibv_create_ah(msg->ep->port->dev->pd, &dest->av); -+ if (!dest->ah) { -+ acm_log(0, "ERROR - failed to create ah\n"); -+ status = ACM_STATUS_ENOMEM; -+ } - } - } - if (!status) { -@@ -1304,59 +1395,6 @@ static void acm_format_mgid(union ibv_gid *mgid, uint16_t pkey, uint8_t tos, - mgid->raw[15] = 0; - } - --static uint64_t acm_path_comp_mask(struct ibv_path_record *path) --{ -- uint32_t fl_hop; -- uint16_t qos_sl; -- uint64_t comp_mask = 0; -- -- acm_log(2, "\n"); -- if (path->service_id) -- comp_mask |= IB_COMP_MASK_PR_SERVICE_ID; -- if (path->dgid.global.interface_id || path->dgid.global.subnet_prefix) -- comp_mask |= IB_COMP_MASK_PR_DGID; -- if (path->sgid.global.interface_id || path->sgid.global.subnet_prefix) -- comp_mask |= IB_COMP_MASK_PR_SGID; -- if (path->dlid) -- comp_mask |= IB_COMP_MASK_PR_DLID; -- if (path->slid) -- comp_mask |= IB_COMP_MASK_PR_SLID; -- -- fl_hop = ntohl(path->flowlabel_hoplimit); -- if (fl_hop >> 8) -- comp_mask |= IB_COMP_MASK_PR_FLOW_LABEL; -- if (fl_hop & 0xFF) -- comp_mask |= IB_COMP_MASK_PR_HOP_LIMIT; -- -- if (path->tclass) -- comp_mask |= IB_COMP_MASK_PR_TCLASS; -- if (path->reversible_numpath & 0x80) -- comp_mask |= IB_COMP_MASK_PR_REVERSIBLE; -- if (path->pkey) -- comp_mask |= IB_COMP_MASK_PR_PKEY; -- -- qos_sl = ntohs(path->qosclass_sl); -- if (qos_sl >> 4) -- comp_mask |= IB_COMP_MASK_PR_QOS_CLASS; -- if (qos_sl & 0xF) -- comp_mask |= IB_COMP_MASK_PR_SL; -- -- if (path->mtu & 0xC0) -- comp_mask |= IB_COMP_MASK_PR_MTU_SELECTOR; -- if (path->mtu & 0x3F) -- comp_mask |= IB_COMP_MASK_PR_MTU; -- if (path->rate & 0xC0) -- comp_mask |= IB_COMP_MASK_PR_RATE_SELECTOR; -- if (path->rate & 0x3F) -- comp_mask |= IB_COMP_MASK_PR_RATE; -- if (path->packetlifetime & 0xC0) -- comp_mask |= IB_COMP_MASK_PR_PACKET_LIFETIME_SELECTOR; -- if (path->packetlifetime & 0x3F) -- comp_mask |= IB_COMP_MASK_PR_PACKET_LIFETIME; -- -- return comp_mask; --} -- - static void acm_init_join(struct ib_sa_mad *mad, union ibv_gid *port_gid, - uint16_t pkey, uint8_t tos, uint8_t tclass, uint8_t sl, uint8_t rate, uint8_t mtu) - { -@@ -1708,6 +1746,15 @@ static uint8_t acm_svr_query_sa(struct acm_ep *ep, struct acm_request *req) - return ACM_STATUS_SUCCESS; - } - -+static int -+acm_is_path_from_port(struct acm_port *port, struct ibv_path_record *path) -+{ -+ if (ib_any_gid(&path->sgid)) { -+ return (port->lid == (ntohs(path->slid) & port->lid_mask)); -+ } -+ return (acm_gid_index(port, &path->sgid) < port->gid_cnt); -+} -+ - static struct acm_ep * - acm_get_ep(struct acm_ep_addr_data *data) - { -@@ -1728,7 +1775,7 @@ acm_get_ep(struct acm_ep_addr_data *data) - port = &dev->port[i]; - - if (data->type == ACM_EP_INFO_PATH && -- port->lid != ntohs(data->info.path.slid)) -+ !acm_is_path_from_port(port, &data->info.path)) - continue; - - for (ep_entry = port->ep_list.Next; ep_entry != &port->ep_list; -@@ -1738,8 +1785,10 @@ acm_get_ep(struct acm_ep_addr_data *data) - if (ep->state != ACM_READY) - continue; - -- if (data->type == ACM_EP_INFO_PATH) -- return ep; // TODO: check pkey -+ if ((data->type == ACM_EP_INFO_PATH) && -+ (!data->info.path.pkey || -+ (ntohs(data->info.path.pkey) == ep->pkey))) -+ return ep; - - if (acm_addr_index(ep, data->info.addr, - (uint8_t) data->type) >= 0) -@@ -1768,12 +1817,6 @@ acm_svr_query(struct acm_client *client, struct acm_resolve_msg *msg) - goto resp; - } - -- if (msg->data[0].type != ACM_EP_INFO_PATH) { -- acm_log(0, "ERROR - unsupported type: 0x%x\n", msg->data[0].type); -- status = ACM_STATUS_EINVAL; -- goto resp; -- } -- - ep = acm_get_ep(&msg->data[0]); - if (!ep) { - acm_log(1, "notice - could not find local end point\n"); -@@ -2060,6 +2103,94 @@ put: - return ret; - } - -+/* -+ * The message buffer contains extra address data buffers. We extract the -+ * destination address from the path record into an extra buffer, so we can -+ * lookup the destination by either LID or GID. -+ */ -+static int -+acm_svr_resolve_path(struct acm_client *client, struct acm_resolve_msg *msg) -+{ -+ struct acm_ep *ep; -+ struct acm_dest *dest; -+ struct ibv_path_record *path; -+ uint8_t *addr; -+ uint8_t status; -+ int ret; -+ -+ acm_log(2, "client %d\n", client->index); -+ if (msg->hdr.length < (ACM_MSG_HDR_LENGTH + ACM_MSG_EP_LENGTH)) { -+ acm_log(0, "notice - invalid msg hdr length %d\n", msg->hdr.length); -+ return acm_client_resolve_resp(client, msg, NULL, ACM_STATUS_EINVAL); -+ } -+ -+ path = &msg->data[0].info.path; -+ if (!path->dlid && ib_any_gid(&path->dgid)) { -+ acm_log(0, "notice - no destination specified\n"); -+ return acm_client_resolve_resp(client, msg, NULL, ACM_STATUS_EDESTADDR); -+ } -+ -+ acm_format_name(2, log_data, sizeof log_data, -+ ACM_EP_INFO_PATH, msg->data[0].info.addr, sizeof *path); -+ acm_log(2, "path %s\n", log_data); -+ ep = acm_get_ep(&msg->data[0]); -+ if (!ep) { -+ acm_log(0, "notice - unknown local end point\n"); -+ return acm_client_resolve_resp(client, msg, NULL, ACM_STATUS_ESRCADDR); -+ } -+ -+ addr = msg->data[1].info.addr; -+ memset(addr, 0, ACM_MAX_ADDRESS); -+ if (path->dlid) { -+ * ((uint16_t *) addr) = path->dlid; -+ dest = acm_acquire_dest(ep, ACM_ADDRESS_LID, addr); -+ } else { -+ memcpy(addr, &path->dgid, sizeof path->dgid); -+ dest = acm_acquire_dest(ep, ACM_ADDRESS_GID, addr); -+ } -+ if (!dest) { -+ acm_log(0, "ERROR - unable to allocate destination in client request\n"); -+ return acm_client_resolve_resp(client, msg, NULL, ACM_STATUS_ENOMEM); -+ } -+ -+ lock_acquire(&dest->lock); -+ switch (dest->state) { -+ case ACM_READY: -+ acm_log(2, "request satisfied from local cache\n"); -+ status = ACM_STATUS_SUCCESS; -+ break; -+ case ACM_INIT: -+ acm_log(2, "have path, bypassing address resolution\n"); -+ acm_record_path_addr(ep, dest, path); -+ /* fall through */ -+ case ACM_ADDR_RESOLVED: -+ acm_log(2, "have address, resolving route\n"); -+ status = acm_resolve_path(ep, dest, acm_dest_sa_resp); -+ if (status) { -+ break; -+ } -+ /* fall through */ -+ default: -+ if (msg->data[0].flags & ACM_FLAGS_NODELAY) { -+ acm_log(2, "lookup initiated, but client wants no delay\n"); -+ status = ACM_STATUS_ENODATA; -+ break; -+ } -+ status = acm_svr_queue_req(dest, client, msg); -+ if (status) { -+ break; -+ } -+ ret = 0; -+ lock_release(&dest->lock); -+ goto put; -+ } -+ lock_release(&dest->lock); -+ ret = acm_client_resolve_resp(client, msg, dest, status); -+put: -+ acm_put_dest(dest); -+ return ret; -+} -+ - static void acm_svr_receive(struct acm_client *client) - { - struct acm_msg msg; -@@ -2085,7 +2216,11 @@ static void acm_svr_receive(struct acm_client *client) - } - - if (resolve_msg->data[0].type == ACM_EP_INFO_PATH) { -- ret = acm_svr_query(client, resolve_msg); -+ if (resolve_msg->data[0].flags & ACM_FLAGS_QUERY_SA) { -+ ret = acm_svr_query(client, resolve_msg); -+ } else { -+ ret = acm_svr_resolve_path(client, resolve_msg); -+ } - } else { - ret = acm_svr_resolve(client, resolve_msg); - } -@@ -2582,7 +2717,7 @@ static void acm_init_port(struct acm_port *port) - break; - } - port->lid = attr.lid; -- port->lid_mask = 0xffff - (1 << attr.lmc) - 1; -+ port->lid_mask = 0xffff - ((1 << attr.lmc) - 1); - - acm_init_dest(&port->sa_dest, ACM_ADDRESS_LID, - (uint8_t *) &attr.sm_lid, sizeof(attr.sm_lid)); -diff --git a/src/acme.c b/src/acme.c -index a7b7413..dd645c1 100644 ---- a/src/acme.c -+++ b/src/acme.c -@@ -493,7 +493,7 @@ static int resolve_lid(struct ibv_path_record *path) - path->dlid = htons((uint16_t) atoi(dest_addr)); - path->reversible_numpath = IBV_PATH_RECORD_REVERSIBLE | 1; - -- ret = ib_acm_resolve_path(path, 0); -+ ret = ib_acm_resolve_path(path, get_resolve_flags()); - if (ret) - printf("ib_acm_resolve_path failed: %s\n", strerror(errno));