From d245664e27148e54469268ad81f41b2a894a131a Mon Sep 17 00:00:00 2001 From: Arlin Davis Date: Fri, 19 Jan 2007 16:21:30 -0800 Subject: [PATCH] uDAPL changes to support exchanging and validation of the device responder_resources and the initiator_depth during connection establishment --- dapl/openib_cma/dapl_ib_cm.c | 36 +++++++++++++++++++++------------- dapl/openib_cma/dapl_ib_util.c | 3 +++ dapl/openib_cma/dapl_ib_util.h | 6 +++--- 3 files changed, 28 insertions(+), 17 deletions(-) mode change 100644 => 100755 dapl/openib_cma/dapl_ib_cm.c mode change 100644 => 100755 dapl/openib_cma/dapl_ib_util.c mode change 100644 => 100755 dapl/openib_cma/dapl_ib_util.h diff --git a/dapl/openib_cma/dapl_ib_cm.c b/dapl/openib_cma/dapl_ib_cm.c old mode 100644 new mode 100755 index 0f24244..8bdd0eb --- a/dapl/openib_cma/dapl_ib_cm.c +++ b/dapl/openib_cma/dapl_ib_cm.c @@ -259,6 +259,18 @@ static struct dapl_cm_id * dapli_req_recv(struct dapl_cm_id *conn, new_conn->sp = conn->sp; new_conn->hca = conn->hca; + /* Get requesters connect data, setup for accept */ + new_conn->params.responder_resources = + DAPL_MIN(event->param.conn.initiator_depth, + conn->hca->ib_trans.max_rdma_rd_in); + new_conn->params.initiator_depth = + DAPL_MIN(event->param.conn.responder_resources, + conn->hca->ib_trans.max_rdma_rd_out); + + new_conn->params.flow_control = event->param.conn.flow_control; + new_conn->params.rnr_retry_count = event->param.conn.rnr_retry_count; + new_conn->params.retry_count = event->param.conn.retry_count; + /* save private data */ if (event->param.conn.private_data_len) { dapl_os_memcpy(new_conn->p_data, @@ -279,7 +291,8 @@ static struct dapl_cm_id * dapli_req_recv(struct dapl_cm_id *conn, event->param.conn.private_data, event->param.conn.private_data_len); dapl_dbg_log(DAPL_DBG_TYPE_CM, " passive_cb: " - "REQ: IP SRC %x PORT %d DST %x PORT %d\n", + "REQ: IP SRC %x PORT %d DST %x PORT %d " + "rr %d init %d\n", ntohl(((struct sockaddr_in *) &ipaddr->src_addr)->sin_addr.s_addr), ntohs(((struct sockaddr_in *) @@ -287,7 +300,9 @@ static struct dapl_cm_id * dapli_req_recv(struct dapl_cm_id *conn, ntohl(((struct sockaddr_in *) &ipaddr->dst_addr)->sin_addr.s_addr), ntohs(((struct sockaddr_in *) - &ipaddr->dst_addr)->sin_port)); + &ipaddr->dst_addr)->sin_port), + new_conn->params.responder_resources, + new_conn->params.initiator_depth); } return new_conn; } @@ -556,8 +571,8 @@ DAT_RETURN dapls_ib_connect(IN DAT_EP_HANDLE ep_handle, /* Setup QP/CM parameters and private data in cm_id */ (void)dapl_os_memzero(&conn->params, sizeof(conn->params)); - conn->params.responder_resources = IB_TARGET_MAX; - conn->params.initiator_depth = IB_INITIATOR_DEPTH; + conn->params.responder_resources = conn->hca->ib_trans.max_rdma_rd_in; + conn->params.initiator_depth = conn->hca->ib_trans.max_rdma_rd_out; conn->params.flow_control = 1; conn->params.rnr_retry_count = IB_RNR_RETRY_COUNT; conn->params.retry_count = IB_RC_RETRY_COUNT; @@ -814,7 +829,6 @@ dapls_ib_accept_connection(IN DAT_CR_HANDLE cr_handle, struct dapl_cm_id *cr_conn = cr_ptr->ib_cm_handle; int ret; DAT_RETURN dat_status; - struct rdma_conn_param conn_params; dapl_dbg_log(DAPL_DBG_TYPE_CM, " accept(cr %p conn %p, id %p, p_data %p, p_sz=%d)\n", @@ -867,16 +881,10 @@ dapls_ib_accept_connection(IN DAT_CR_HANDLE cr_handle, ep_ptr->qp_handle = cr_conn; ep_ptr->cm_handle = cr_conn; cr_conn->ep = ep_ptr; + cr_conn->params.private_data = p_data; + cr_conn->params.private_data_len = p_size; - memset(&conn_params, 0, sizeof(conn_params)); - conn_params.private_data = p_data; - conn_params.private_data_len = p_size; - conn_params.responder_resources = IB_TARGET_MAX; - conn_params.initiator_depth = IB_INITIATOR_DEPTH; - conn_params.flow_control = 1; - conn_params.rnr_retry_count = IB_RNR_RETRY_COUNT; - - ret = rdma_accept(cr_conn->cm_id, &conn_params); + ret = rdma_accept(cr_conn->cm_id, &cr_conn->params); if (ret) { dapl_dbg_log(DAPL_DBG_TYPE_ERR," accept: ERROR %d\n", ret); dat_status = dapl_convert_errno(ret, "accept"); diff --git a/dapl/openib_cma/dapl_ib_util.c b/dapl/openib_cma/dapl_ib_util.c old mode 100644 new mode 100755 index 6bb35f6..0606312 --- a/dapl/openib_cma/dapl_ib_util.c +++ b/dapl/openib_cma/dapl_ib_util.c @@ -469,6 +469,9 @@ DAT_RETURN dapls_ib_query_hca(IN DAPL_HCA *hca_ptr, ia_attr->num_vendor_attr = 0; ia_attr->vendor_attr = NULL; ia_attr->max_iov_segments_per_rdma_read = dev_attr.max_sge; + /* save rd_atom for peer validation during connect requests */ + hca_ptr->ib_trans.max_rdma_rd_in = dev_attr.max_qp_rd_atom; + hca_ptr->ib_trans.max_rdma_rd_out = dev_attr.max_qp_rd_atom; dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " query_hca: (ver=%x) ep %d ep_q %d evd %d evd_q %d\n", diff --git a/dapl/openib_cma/dapl_ib_util.h b/dapl/openib_cma/dapl_ib_util.h old mode 100644 new mode 100755 index 1fadb08..457d26b --- a/dapl/openib_cma/dapl_ib_util.h +++ b/dapl/openib_cma/dapl_ib_util.h @@ -73,10 +73,7 @@ typedef ib_hca_handle_t dapl_ibal_ca_t; #define IB_ARP_RETRY_COUNT 15 /* 60 sec total */ #define IB_ROUTE_TIMEOUT 4000 /* 4 sec */ #define IB_ROUTE_RETRY_COUNT 15 /* 60 sec total */ -#define IB_REQ_MRA_TIMEOUT 27 /* a little over 9 minutes */ #define IB_MAX_AT_RETRY 3 -#define IB_TARGET_MAX 4 /* max_qp_ous_rd_atom */ -#define IB_INITIATOR_DEPTH 4 /* max_qp_init_rd_atom */ typedef enum { IB_CME_CONNECTED, @@ -272,6 +269,9 @@ typedef struct _ib_hca_transport ib_async_qp_handler_t async_qp_error; uint8_t max_cm_timeout; uint8_t max_cm_retries; + /* device attributes */ + int max_rdma_rd_in; + int max_rdma_rd_out; } ib_hca_transport_t; -- 2.41.0