#include "dapl_ib_util.h"
#include <sys/poll.h>
#include <signal.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
#include <rdma/rdma_cma_ib.h>
extern struct rdma_event_channel *g_cm_events;
&ipaddr->src_addr)->sin_addr.s_addr),
ntohl(((struct sockaddr_in *)
&ipaddr->dst_addr)->sin_addr.s_addr));
-
- ret = rdma_resolve_route(conn->cm_id, 2000);
+
+ ret = rdma_resolve_route(conn->cm_id, conn->route_timeout);
if (ret) {
dapl_dbg_log(DAPL_DBG_TYPE_ERR,
" rdma_connect failed: %s\n",strerror(errno));
struct rdma_addr *ipaddr = &conn->cm_id->route.addr;
struct ib_addr *ibaddr = &conn->cm_id->route.addr.addr.ibaddr;
#endif
+
dapl_dbg_log(DAPL_DBG_TYPE_CM,
" route_resolve: cm_id %p SRC %x DST %x PORT %d\n",
conn->cm_id,
case RDMA_CM_EVENT_UNREACHABLE:
case RDMA_CM_EVENT_CONNECT_ERROR:
{
- ib_cm_events_t cm_event;
- dapl_dbg_log(
+ dapl_dbg_log(
DAPL_DBG_TYPE_WARN,
" dapli_cm_active_handler: CONN_ERR "
" event=0x%x status=%d %s\n",
event->event, event->status,
(event->status == -ETIMEDOUT)?"TIMEOUT":"" );
- /* no device type specified so assume IB for now */
- if (event->status == -ETIMEDOUT) /* IB timeout */
- cm_event = IB_CME_TIMEOUT;
- else
- cm_event = IB_CME_DESTINATION_UNREACHABLE;
-
- dapl_evd_connection_callback(conn, cm_event, NULL, conn->ep);
+ /* per DAT SPEC provider always returns UNREACHABLE */
+ dapl_evd_connection_callback(conn,
+ IB_CME_DESTINATION_UNREACHABLE,
+ NULL, conn->ep);
break;
}
case RDMA_CM_EVENT_REJECTED:
break;
case RDMA_CM_EVENT_DISCONNECTED:
+ rdma_disconnect(conn->cm_id); /* required for DREP */
/* validate EP handle */
if (!DAPL_BAD_HANDLE(conn->ep, DAPL_MAGIC_EP))
dapl_evd_connection_callback(conn,
break;
case RDMA_CM_EVENT_DISCONNECTED:
+ rdma_disconnect(conn->cm_id); /* required for DREP */
/* validate SP handle context */
if (!DAPL_BAD_HANDLE(conn->sp, DAPL_MAGIC_PSP) ||
!DAPL_BAD_HANDLE(conn->sp, DAPL_MAGIC_RSP))
IN void *p_data)
{
struct dapl_ep *ep_ptr = ep_handle;
-
+ struct dapl_cm_id *conn;
+
/* Sanity check */
if (NULL == ep_ptr)
return DAT_SUCCESS;
r_qual,p_data,p_size);
/* rdma conn and cm_id pre-bound; reference via qp_handle */
- ep_ptr->cm_handle = ep_ptr->qp_handle;
+ conn = ep_ptr->cm_handle = ep_ptr->qp_handle;
/* Setup QP/CM parameters and private data in cm_id */
- (void)dapl_os_memzero(&ep_ptr->cm_handle->params,
- sizeof(ep_ptr->cm_handle->params));
- ep_ptr->cm_handle->params.responder_resources = IB_TARGET_MAX;
- ep_ptr->cm_handle->params.initiator_depth = IB_INITIATOR_DEPTH;
- ep_ptr->cm_handle->params.flow_control = 1;
- ep_ptr->cm_handle->params.rnr_retry_count = IB_RNR_RETRY_COUNT;
- ep_ptr->cm_handle->params.retry_count = IB_RC_RETRY_COUNT;
+ (void)dapl_os_memzero(&conn->params, sizeof(conn->params));
+ conn->params.responder_resources = IB_TARGET_MAX;
+ conn->params.initiator_depth = IB_INITIATOR_DEPTH;
+ conn->params.flow_control = 1;
+ conn->params.rnr_retry_count = IB_RNR_RETRY_COUNT;
+ conn->params.retry_count = IB_RC_RETRY_COUNT;
if (p_size) {
- dapl_os_memcpy(ep_ptr->cm_handle->p_data, p_data, p_size);
- ep_ptr->cm_handle->params.private_data =
- ep_ptr->cm_handle->p_data;
- ep_ptr->cm_handle->params.private_data_len = p_size;
+ dapl_os_memcpy(conn->p_data, p_data, p_size);
+ conn->params.private_data = conn->p_data;
+ conn->params.private_data_len = p_size;
}
+ /* copy in remote address, need a copy for retry attempts */
+ dapl_os_memcpy(&conn->r_addr, r_addr, sizeof(*r_addr));
+
/* Resolve remote address, src already bound during QP create */
- ((struct sockaddr_in*)r_addr)->sin_port = htons(MAKE_PORT(r_qual));
- if (rdma_resolve_addr(ep_ptr->cm_handle->cm_id,
- NULL, (struct sockaddr *)r_addr, 2000))
+ ((struct sockaddr_in*)&conn->r_addr)->sin_port = htons(MAKE_PORT(r_qual));
+ ((struct sockaddr_in*)&conn->r_addr)->sin_family = AF_INET;
+
+ if (rdma_resolve_addr(conn->cm_id, NULL,
+ (struct sockaddr *)&conn->r_addr,
+ conn->arp_timeout))
return dapl_convert_errno(errno,"ib_connect");
dapl_dbg_log(DAPL_DBG_TYPE_CM,
- " connect: resolve_addr: cm_id %p SRC %x DST %x port %d\n",
- ep_ptr->cm_handle->cm_id,
- ntohl(((struct sockaddr_in *)
- &ep_ptr->cm_handle->hca->hca_address)->sin_addr.s_addr),
- ntohl(((struct sockaddr_in *)r_addr)->sin_addr.s_addr),
- MAKE_PORT(r_qual) );
+ " connect: resolve_addr: cm_id %p -> %s port %d\n",
+ conn->cm_id,
+ inet_ntoa(((struct sockaddr_in *)&conn->r_addr)->sin_addr),
+ ((struct sockaddr_in*)&conn->r_addr)->sin_port );
return DAT_SUCCESS;
}
case RDMA_CM_EVENT_ADDR_RESOLVED:
dapli_addr_resolve(conn);
break;
+
case RDMA_CM_EVENT_ROUTE_RESOLVED:
dapli_route_resolve(conn);
break;
+
case RDMA_CM_EVENT_ADDR_ERROR:
+ dapl_dbg_log(DAPL_DBG_TYPE_WARN,
+ " CM ADDR ERROR: -> %s retry (%d)..\n",
+ inet_ntoa(((struct sockaddr_in *)
+ &conn->r_addr)->sin_addr),
+ conn->arp_retries);
+
+ /* retry address resolution */
+ if ((--conn->arp_retries) &&
+ (event->status == -ETIMEDOUT)) {
+ int ret;
+ ret = rdma_resolve_addr(
+ conn->cm_id, NULL,
+ (struct sockaddr *)&conn->r_addr,
+ conn->arp_timeout);
+ if (!ret)
+ break;
+ else {
+ dapl_dbg_log(
+ DAPL_DBG_TYPE_WARN,
+ " ERROR: rdma_resolve_addr = "
+ "%d %s\n",
+ ret,strerror(errno));
+ }
+ }
+ /* retries exhausted or resolve_addr failed */
+ dapl_evd_connection_callback(
+ conn, IB_CME_DESTINATION_UNREACHABLE,
+ NULL, conn->ep);
+ break;
+
+
case RDMA_CM_EVENT_ROUTE_ERROR:
- dapl_evd_connection_callback(conn,
- IB_CME_DESTINATION_UNREACHABLE,
- NULL, conn->ep);
+ dapl_dbg_log(DAPL_DBG_TYPE_WARN,
+ " CM ROUTE ERROR: -> %s retry (%d)..\n",
+ inet_ntoa(((struct sockaddr_in *)
+ &conn->r_addr)->sin_addr),
+ conn->route_retries );
+
+ /* retry route resolution */
+ if ((--conn->route_retries) &&
+ (event->status == -ETIMEDOUT))
+ dapli_addr_resolve(conn);
+ else
+ dapl_evd_connection_callback( conn,
+ IB_CME_DESTINATION_UNREACHABLE,
+ NULL, conn->ep);
break;
+
case RDMA_CM_EVENT_DEVICE_REMOVAL:
dapl_evd_connection_callback(conn,
IB_CME_LOCAL_FAILURE,
/* Setup attributes and create qp */
dapl_os_memzero((void*)&qp_create, sizeof(qp_create));
qp_create.cap.max_send_wr = attr->max_request_dtos;
- qp_create.cap.max_recv_wr = attr->max_recv_dtos;
qp_create.cap.max_send_sge = attr->max_request_iov;
- qp_create.cap.max_recv_sge = attr->max_recv_iov;
qp_create.cap.max_inline_data =
ia_ptr->hca_ptr->ib_trans.max_inline_send;
qp_create.send_cq = req_cq;
- qp_create.recv_cq = rcv_cq;
+
+ /* ibv assumes rcv_cq is never NULL, set to req_cq */
+ if (rcv_cq == NULL) {
+ qp_create.recv_cq = req_cq;
+ qp_create.cap.max_recv_wr = 0;
+ qp_create.cap.max_recv_sge = 0;
+ } else {
+ qp_create.recv_cq = rcv_cq;
+ qp_create.cap.max_recv_wr = attr->max_recv_dtos;
+ qp_create.cap.max_recv_sge = attr->max_recv_iov;
+ }
qp_create.qp_type = IBV_QPT_RC;
qp_create.qp_context = (void*)ep_ptr;
conn->cm_id = cm_id;
conn->ep = ep_ptr;
conn->hca = ia_ptr->hca_ptr;
+
+ /* setup timers for address and route resolution */
+ conn->arp_timeout = dapl_os_get_env_val("DAPL_CM_ARP_TIMEOUT_MS",
+ IB_ARP_TIMEOUT);
+ conn->arp_retries = dapl_os_get_env_val("DAPL_CM_ARP_RETRY_COUNT",
+ IB_ARP_RETRY_COUNT);
+ conn->route_timeout = dapl_os_get_env_val("DAPL_CM_ROUTE_TIMEOUT_MS",
+ IB_ROUTE_TIMEOUT);
+ conn->route_retries = dapl_os_get_env_val("DAPL_CM_ROUTE_RETRY_COUNT",
+ IB_ROUTE_RETRY_COUNT);
+
ep_ptr->qp_handle = conn;
ep_ptr->qp_state = IB_QP_STATE_INIT;