*
* Description:
*
- * The uDAPL openib provider - init, open, close, utilities
+ * The uDAPL openib provider - init, open, close, utilities, work thread
*
****************************************************************************
* Source Control System Information
int g_dapl_loopback_connection = 0;
-/* get lid */
-int dapli_get_lid(struct dapl_hca *hca_ptr, int port, uint16_t *lid )
-{
- struct ibv_port_attr attr;
-
- if (ibv_query_port(hca_ptr->ib_hca_handle, port, &attr))
- return 1;
-
- *lid = attr.lid;
-
- return 0;
-}
-
-/* get gid */
-int dapli_get_gid(struct dapl_hca *hca_ptr, int port,
- int index, union ibv_gid *gid )
-{
- /* ibv_query_gid() coming soon, until then HACK */
- char path[128];
- char val[40];
- char name[256];
- char *token;
- uint16_t *p_gid;
-
- if (sysfs_get_mnt_path(path, sizeof path)) {
- fprintf(stderr, "Couldn't find sysfs mount.\n");
- return 1;
- }
- sprintf(name, "%s/class/infiniband/%s/ports/%d/gids/%d", path,
- ibv_get_device_name(hca_ptr->ib_trans.ib_dev), port, index);
-
- if (sysfs_read_attribute_value(name, val, sizeof val)) {
- fprintf(stderr, "Couldn't read GID at %s\n", name);
- return 1;
- }
-
- /* get token strings with delimiter */
- token = strtok(val,":");
- p_gid = (uint16_t*)gid->raw;
- while (token) {
- *p_gid = strtoul(token,NULL,16);
- *p_gid = htons(*p_gid); /* convert each token to network order */
- token = strtok(NULL,":");
- p_gid++;
- }
- return 0;
-}
-
/* just get IP address, IPv4 only for now */
int dapli_get_hca_addr( struct dapl_hca *hca_ptr )
dapli_ip_comp_handler(at_rec.req_id, (void*)&at_rec, status);
} else {
dat_status = dapl_os_wait_object_wait(&hca_ptr->ib_trans.wait_object,500000);
- return 0;
if (dat_status != DAT_SUCCESS)
ib_at_cancel(at_rec.req_id);
}
/* set inline max with enviromment or default, get local lid and gid 0 */
hca_ptr->ib_trans.max_inline_send =
- dapl_os_get_env_val ( "DAPL_MAX_INLINE", INLINE_SEND_DEFAULT );
-
- if (dapli_get_lid(hca_ptr, hca_ptr->port_num,
- &hca_ptr->ib_trans.lid)) {
- dapl_dbg_log (DAPL_DBG_TYPE_ERR,
- " open_hca: IB get LID failed for %s\n",
- ibv_get_device_name(hca_ptr->ib_trans.ib_dev) );
- goto bail;
- }
+ dapl_os_get_env_val("DAPL_MAX_INLINE", INLINE_SEND_DEFAULT);
- if (dapli_get_gid(hca_ptr, hca_ptr->port_num, 0,
- &hca_ptr->ib_trans.gid)) {
+ /* GID with port_num provided, index 0 for now */
+ if (ibv_query_gid(hca_ptr->ib_hca_handle,
+ hca_ptr->port_num, 0, &hca_ptr->ib_trans.gid)) {
dapl_dbg_log (DAPL_DBG_TYPE_ERR,
" open_hca: IB get GID failed for %s\n",
- ibv_get_device_name(hca_ptr->ib_trans.ib_dev) );
+ ibv_get_device_name(hca_ptr->ib_trans.ib_dev));
goto bail;
}
-
- dapl_dbg_log(DAPL_DBG_TYPE_CM,
- " open_hca: LID 0x%x GID subnet %016llx id %016llx\n",
- hca_ptr->ib_trans.lid,
+
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
+ " open_hca: GID subnet %016llx id %016llx\n",
(unsigned long long)bswap_64(hca_ptr->ib_trans.gid.global.subnet_prefix),
(unsigned long long)bswap_64(hca_ptr->ib_trans.gid.global.interface_id) );
ibv_close_device(hca_ptr->ib_hca_handle);
hca_ptr->ib_hca_handle = IB_INVALID_HANDLE;
return DAT_INTERNAL_ERROR;
-
}
*/
DAT_RETURN dapls_ib_close_hca ( IN DAPL_HCA *hca_ptr )
{
- dapl_dbg_log (DAPL_DBG_TYPE_UTIL," close_hca: %p\n",hca_ptr);
+ dapl_dbg_log (DAPL_DBG_TYPE_UTIL," close_hca: %p->%p\n",
+ hca_ptr,hca_ptr->ib_hca_handle);
+
+ dapli_cq_thread_destroy(hca_ptr);
if (hca_ptr->ib_hca_handle != IB_INVALID_HANDLE) {
- dapli_cq_thread_destroy(hca_ptr);
if (ibv_close_device(hca_ptr->ib_hca_handle))
return(dapl_convert_errno(errno,"ib_close_device"));
hca_ptr->ib_hca_handle = IB_INVALID_HANDLE;
OUT DAT_EP_ATTR *ep_attr,
OUT DAT_SOCK_ADDR6 *ip_addr)
{
- DAT_RETURN dat_status = DAT_SUCCESS;
+ struct ibv_device_attr dev_attr;
+ struct ibv_port_attr port_attr;
if (hca_ptr->ib_hca_handle == NULL) {
dapl_dbg_log (DAPL_DBG_TYPE_ERR," query_hca: BAD handle\n");
if (ip_addr != NULL)
memcpy(ip_addr, &hca_ptr->hca_address, sizeof(DAT_SOCK_ADDR6));
+ if (ia_attr == NULL && ep_attr == NULL)
+ return DAT_SUCCESS;
+
+ /* query verbs for this device and port attributes */
+ if (ibv_query_device(hca_ptr->ib_hca_handle, &dev_attr) ||
+ ibv_query_port(hca_ptr->ib_hca_handle,
+ hca_ptr->port_num, &port_attr))
+ return(dapl_convert_errno(errno,"ib_query_hca"));
+
if (ia_attr != NULL) {
ia_attr->adapter_name[DAT_NAME_MAX_LENGTH - 1] = '\0';
ia_attr->vendor_name[DAT_NAME_MAX_LENGTH - 1] = '\0';
((struct sockaddr_in *)ia_attr->ia_address_ptr)->sin_addr.s_addr >> 8 & 0xff,
((struct sockaddr_in *)ia_attr->ia_address_ptr)->sin_addr.s_addr >> 16 & 0xff,
((struct sockaddr_in *)ia_attr->ia_address_ptr)->sin_addr.s_addr >> 24 & 0xff );
+
+ ia_attr->hardware_version_major = dev_attr.hw_ver;
+ ia_attr->hardware_version_minor = dev_attr.fw_ver;
+ ia_attr->max_eps = dev_attr.max_qp;
+ ia_attr->max_dto_per_ep = dev_attr.max_qp_wr;
+ ia_attr->max_rdma_read_per_ep = dev_attr.max_qp_rd_atom;
+ ia_attr->max_evds = dev_attr.max_cq;
+ ia_attr->max_evd_qlen = dev_attr.max_cqe;
+ ia_attr->max_iov_segments_per_dto = dev_attr.max_sge;
+ ia_attr->max_lmrs = dev_attr.max_mr;
+ ia_attr->max_lmr_block_size = dev_attr.max_mr_size;
+ ia_attr->max_rmrs = dev_attr.max_mw;
+ ia_attr->max_lmr_virtual_address = dev_attr.max_mr_size;
+ ia_attr->max_rmr_target_address = dev_attr.max_mr_size;
+ ia_attr->max_pzs = dev_attr.max_pd;
+ ia_attr->max_mtu_size = port_attr.max_msg_sz;
+ ia_attr->max_rdma_size = port_attr.max_msg_sz;
+ ia_attr->num_transport_attr = 0;
+ ia_attr->transport_attr = NULL;
+ ia_attr->num_vendor_attr = 0;
+ ia_attr->vendor_attr = NULL;
- /* TODO: need verbs query call */
- ia_attr->max_eps = 64000;
- ia_attr->max_dto_per_ep = 64000;
- ia_attr->max_rdma_read_per_ep = 8;
- ia_attr->max_evds = 64000;
- ia_attr->max_evd_qlen = 64000;
- ia_attr->max_iov_segments_per_dto = 32;
- ia_attr->max_lmrs = 64000;
- ia_attr->max_lmr_block_size = 0x80000000;
- ia_attr->max_rmrs = 64000;
- ia_attr->max_lmr_virtual_address = 0x80000000;
- ia_attr->max_rmr_target_address = 0x80000000;
- ia_attr->max_pzs = 64000;
- ia_attr->max_mtu_size = 0x80000000;
- ia_attr->max_rdma_size = 0x80000000;
- ia_attr->num_transport_attr = 0;
- ia_attr->transport_attr = NULL;
- ia_attr->num_vendor_attr = 0;
- ia_attr->vendor_attr = NULL;
-
- dapl_dbg_log (DAPL_DBG_TYPE_UTIL,
- " query_hca: (%d.%d) ep %d ep_q %d evd %d evd_q %d\n",
- ia_attr->hardware_version_major,
- ia_attr->hardware_version_minor,
- ia_attr->max_eps, ia_attr->max_dto_per_ep,
- ia_attr->max_evds, ia_attr->max_evd_qlen );
- dapl_dbg_log (DAPL_DBG_TYPE_UTIL,
- " query_hca: msg %llu rdma %llu iov %d lmr %d rmr %d\n",
- ia_attr->max_mtu_size, ia_attr->max_rdma_size,
- ia_attr->max_iov_segments_per_dto, ia_attr->max_lmrs,
- ia_attr->max_rmrs );
+ dapl_dbg_log (DAPL_DBG_TYPE_UTIL,
+ " query_hca: (%x.%x) ep %d ep_q %d evd %d evd_q %d\n",
+ ia_attr->hardware_version_major,
+ ia_attr->hardware_version_minor,
+ ia_attr->max_eps, ia_attr->max_dto_per_ep,
+ ia_attr->max_evds, ia_attr->max_evd_qlen );
+ dapl_dbg_log (DAPL_DBG_TYPE_UTIL,
+ " query_hca: msg %llu rdma %llu iov %d lmr %d rmr %d\n",
+ ia_attr->max_mtu_size, ia_attr->max_rdma_size,
+ ia_attr->max_iov_segments_per_dto, ia_attr->max_lmrs,
+ ia_attr->max_rmrs );
}
- /* TODO: need verbs query call */
+
if (ep_attr != NULL) {
- ep_attr->max_mtu_size = 0x80000000;
- ep_attr->max_rdma_size = 0x80000000;
- ep_attr->max_recv_dtos = 64000;
- ep_attr->max_request_dtos = 64000;
- ep_attr->max_recv_iov = 32;
- ep_attr->max_request_iov = 32;
- ep_attr->max_rdma_read_in = 8;
- ep_attr->max_rdma_read_out= 8;
+ ep_attr->max_mtu_size = port_attr.max_msg_sz;
+ ep_attr->max_rdma_size = port_attr.max_msg_sz;
+ ep_attr->max_recv_dtos = dev_attr.max_qp_wr;
+ ep_attr->max_request_dtos = dev_attr.max_qp_wr;
+ ep_attr->max_recv_iov = dev_attr.max_sge;
+ ep_attr->max_request_iov = dev_attr.max_sge;
+ ep_attr->max_rdma_read_in = dev_attr.max_qp_rd_atom;
+ ep_attr->max_rdma_read_out= dev_attr.max_qp_rd_atom;
dapl_dbg_log (DAPL_DBG_TYPE_UTIL,
" query_hca: MAX msg %llu dto %d iov %d rdma i%d,o%d\n",
ep_attr->max_mtu_size,
ep_attr->max_rdma_read_in, ep_attr->max_rdma_read_out);
}
- return dat_status;
+ return DAT_SUCCESS;
}
/*
projects / ~ardavis / dapl.git / commitdiff