From 1f4baf860cf2c17960885df7ff49cc0021fe317e Mon Sep 17 00:00:00 2001 From: Arlin Davis Date: Fri, 16 Oct 2015 10:21:19 -0700 Subject: [PATCH] scm: CONN_PENDING: SOCKOPT ERR Connection refused -> Error caused by cm_msg size compatability issue with new v8 protocol and older socket cm providers (2.1.4 and older). The ucm, cma, and mcm providers are not affected. Modify socket data sizes for SCM request/reply to interoperate between new v8 with smaller private data and older protocols. Adjust SCM reply/rtu based on remote CM version and retry a failed request with pre-v8 adjusted size in case of server side failure. Signed-off-by: Arlin Davis --- dapl/openib_scm/cm.c | 55 +++++++++++++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 16 deletions(-) diff --git a/dapl/openib_scm/cm.c b/dapl/openib_scm/cm.c index ecc5418..6ee99f1 100644 --- a/dapl/openib_scm/cm.c +++ b/dapl/openib_scm/cm.c @@ -63,6 +63,8 @@ extern char *gid_str; +#define SCM_BC_DIFF 2 /* cm_msg adjust, pre v8 */ + /* forward declarations */ static DAT_RETURN dapli_socket_connect(DAPL_EP * ep_ptr, @@ -561,6 +563,11 @@ static void dapli_socket_connected(dp_ib_cm_handle_t cm_ptr, int err) /* send qp info and pdata to remote peer */ exp = sizeof(ib_cm_msg_t) - DCM_MAX_PDATA_SIZE; + if (cm_ptr->retry == SCM_CR_RETRY-1) { + exp += SCM_BC_DIFF; /* retry with pre-v8 req */ + dapl_log(DAPL_DBG_TYPE_CM_WARN, + " CONN_REQ: retry pre-v8 protocol; peer != v8\n"); + } iov[0].iov_base = (void *)&cm_ptr->msg; iov[0].iov_len = exp; if (cm_ptr->msg.p_size) { @@ -583,16 +590,14 @@ static void dapli_socket_connected(dp_ib_cm_handle_t cm_ptr, int err) goto bail; } - dapl_dbg_log(DAPL_DBG_TYPE_CM, - " CONN_PENDING: sending SRC lid=0x%x," - " qpn=0x%x, psize=%d\n", - ntohs(cm_ptr->msg.saddr.ib.lid), - ntohl(cm_ptr->msg.saddr.ib.qpn), - ntohs(cm_ptr->msg.p_size)); - dapl_dbg_log(DAPL_DBG_TYPE_CM, - " CONN_PENDING: SRC GID %s\n", - inet_ntop(AF_INET6, &cm_ptr->hca->ib_trans.gid, - gid_str, sizeof(gid_str))); + dapl_log(DAPL_DBG_TYPE_CM, + " CONN_REQ: (%d) SRC lid=0x%x," + " qpn=0x%x, psize=%d GID %s\n", + exp, ntohs(cm_ptr->msg.saddr.ib.lid), + ntohl(cm_ptr->msg.saddr.ib.qpn), + ntohs(cm_ptr->msg.p_size), + inet_ntop(AF_INET6, &cm_ptr->hca->ib_trans.gid, + gid_str, sizeof(gid_str))); DAPL_CNTR(((DAPL_IA *)dapl_llist_peek_head(&cm_ptr->hca->ia_list_head)), DCNT_IA_CM_REQ_TX); return; @@ -730,9 +735,16 @@ static void dapli_socket_connect_rtu(dp_ib_cm_handle_t cm_ptr) socklen_t sl; /* read DST information into cm_ptr, overwrite SRC info */ - dapl_dbg_log(DAPL_DBG_TYPE_EP, " connect_rtu: recv peer QP data\n"); - len = recv(cm_ptr->socket, (char *)&cm_ptr->msg, exp, 0); + + if (ntohs(cm_ptr->msg.ver) < DCM_VER_XPS) { + len += recv(cm_ptr->socket, (char *)&cm_ptr->msg+len, SCM_BC_DIFF, 0); + exp += SCM_BC_DIFF; + } + + dapl_log(DAPL_DBG_TYPE_CM, " CONN_REP_IN: ver %d cm_sz %d, p_sz %d\n", + ntohs(cm_ptr->msg.ver), exp, ntohs(cm_ptr->msg.p_size)); + if (len != exp || ntohs(cm_ptr->msg.ver) < DCM_VER_MIN) { int err = dapl_socket_errno(); dapl_log(DAPL_DBG_TYPE_CM_WARN, @@ -745,9 +757,10 @@ static void dapli_socket_connect_rtu(dp_ib_cm_handle_t cm_ptr) cm_ptr->retry); /* Retry; corner case where server tcp stack resets under load */ - if (err == ECONNRESET && --cm_ptr->retry) { + if ((err == ECONNRESET && --cm_ptr->retry) || (--cm_ptr->retry == SCM_CR_RETRY-1)) { closesocket(cm_ptr->socket); cm_ptr->socket = DAPL_INVALID_SOCKET; + dapl_log(DAPL_DBG_TYPE_CM_WARN, "CONN_REQ: retry %d\n", cm_ptr->retry); dapli_socket_connect(cm_ptr->ep, (DAT_IA_ADDRESS_PTR)&cm_ptr->addr, ntohs(((struct sockaddr_in *)&cm_ptr->addr)->sin_port) - 1000, ntohs(cm_ptr->msg.p_size), &cm_ptr->msg.p_data, cm_ptr->retry); @@ -800,9 +813,10 @@ static void dapli_socket_connect_rtu(dp_ib_cm_handle_t cm_ptr) len = recv(cm_ptr->socket, cm_ptr->msg.p_data, exp, 0); if (len != exp) { int err = dapl_socket_errno(); - dapl_log(DAPL_DBG_TYPE_ERR, - " CONN_RTU read pdata: ERR 0x%x %s, rcnt=%d -> %s\n", - err, strerror(err), len, + dapl_log(DAPL_DBG_TYPE_CM_WARN, + " CONN_RTU read pdata: ERR 0x%x %s" + " rcv %d != exp %d -> %s\n", + err, strerror(err), len, exp, inet_ntoa(((struct sockaddr_in *) ep_ptr->param. remote_ia_address_ptr)->sin_addr)); @@ -1121,6 +1135,12 @@ static void dapli_socket_accept_data(ib_cm_srvc_handle_t acm_ptr) /* read in DST QP info, IA address. check for private data */ len = recv(acm_ptr->socket, (char *)&acm_ptr->msg, exp, 0); + + if (ntohs(acm_ptr->msg.ver) < DCM_VER_XPS) { + len += recv(acm_ptr->socket, (char *)&acm_ptr->msg+len, SCM_BC_DIFF, 0); + exp += SCM_BC_DIFF; + } + if (len != exp || ntohs(acm_ptr->msg.ver) < DCM_VER_MIN) { int err = dapl_socket_errno(); dapl_log(DAPL_DBG_TYPE_ERR, @@ -1251,6 +1271,9 @@ dapli_socket_accept_usr(DAPL_EP * ep_ptr, DAPL_MIN(ep_ptr->param.ep_attr.max_rdma_read_out, cm_ptr->msg.rd_in); + if (ntohs(cm_ptr->msg.ver) < DCM_VER_XPS) + exp += SCM_BC_DIFF; + /* modify QP to RTR and then to RTS with remote info already read */ dapl_os_lock(&ep_ptr->header.lock); if (dapls_modify_qp_state(ep_ptr->qp_handle->qp, -- 2.46.0