DAPL_ASYNC_QP_ERROR
} DAPL_ASYNC_HANDLER_TYPE;
+typedef enum dapl_open_flags
+{
+ DAPL_OPEN_NORMAL,
+ DAPL_OPEN_QUERY
+} DAPL_OPEN_FLAGS;
+
int dapls_ib_init (void);
DAT_RETURN dapls_ib_open_hca (
IN char *namestr,
- IN DAPL_HCA *hca_ptr);
+ IN DAPL_HCA *hca_ptr,
+ IN DAPL_OPEN_FLAGS flags);
DAT_RETURN dapls_ib_close_hca (
IN DAPL_HCA *hca_ptr);
dapl_os_lock(&hca_ptr->lock);
if (hca_ptr->ib_hca_handle == IB_INVALID_HANDLE) {
/* register with the HW */
- dat_status = dapls_ib_open_hca(hca_ptr->name, hca_ptr);
+ dat_status = dapls_ib_open_hca(hca_ptr->name, hca_ptr, DAPL_OPEN_NORMAL);
if (dat_status != DAT_SUCCESS) {
dapl_dbg_log(DAPL_DBG_TYPE_ERR,
* Obtain parameters from the HCA. Protect against multiple
* IAs beating on the HCA at the same time.
*/
- dat_status =
- dapls_ib_query_hca(ia_ptr->hca_ptr, ia_attr, NULL, NULL);
+ dat_status = dapls_ib_query_hca(ia_ptr->hca_ptr,
+ ia_attr, NULL, NULL);
if (dat_status != DAT_SUCCESS) {
goto bail;
}
#ifdef DAPL_COUNTERS
{
DAPL_IA *ia = (DAPL_IA *)dapl_llist_peek_head(&hca_ptr->ia_list_head);
- dapli_stop_counters(ia);
+ if (hca_ptr->ib_hca_handle)
+ dapli_stop_counters(ia);
dapl_os_free(ia->cntrs, sizeof(DAT_UINT64) * DCNT_IA_ALL_COUNTERS);
}
#endif
*
*/
DAT_RETURN dapls_ib_open_hca ( IN char *hca_name,
- IN DAPL_HCA *p_hca )
+ IN DAPL_HCA *p_hca,
+ IN DAPL_OPEN_FLAGS flags)
{
dapl_ibal_ca_t *p_ca;
IB_HCA_NAME ca_guid;
char *mtu_str;
char *mode_str;
char *read_str;
+ char *port_state_str;
#ifdef DAT_IB_COLLECTIVES
/* Collective member device and address information */
ib_thread_state_t coll_thread_state;
* dapl_convert_errno
*
*/
-DAT_RETURN dapls_ib_open_hca(IN IB_HCA_NAME hca_name, IN DAPL_HCA * hca_ptr)
+DAT_RETURN dapls_ib_open_hca(IN IB_HCA_NAME hca_name,
+ IN DAPL_HCA * hca_ptr,
+ IN DAPL_OPEN_FLAGS flags)
{
struct rdma_cm_id *cm_id = NULL;
union ibv_gid *gid;
int ret;
- DAT_RETURN dat_status;
+ DAT_RETURN dat_status = DAT_SUCCESS;
- dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
- " open_hca: %s - %p\n", hca_name, hca_ptr);
+ /* HCA name will be hostname or IP address */
+ if (getipaddr((char *)hca_name,
+ (char *)&hca_ptr->hca_address,
+ sizeof(DAT_SOCK_ADDR6)))
+ return DAT_INVALID_ADDRESS;
+
+ if (flags & DAPL_OPEN_QUERY) {
+ dapl_log(DAPL_DBG_TYPE_WARN,
+ " WARNING! open_hca: %s %s - %p in %s\n",
+ PROVIDER_NAME, hca_name, hca_ptr,
+ flags & DAPL_OPEN_QUERY ? "QUERY MODE":"");
+ goto done;
+ }
/* Setup the global cm event channel */
dapl_os_lock(&g_hca_lock);
dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
" open_hca: RDMA channel created (%p)\n", g_cm_events);
- /* HCA name will be hostname or IP address */
- if (getipaddr((char *)hca_name,
- (char *)&hca_ptr->hca_address,
- sizeof(DAT_SOCK_ADDR6)))
- return DAT_INVALID_ADDRESS;
-
/* cm_id will bind local device/GID based on IP address */
if (rdma_create_id(g_cm_events, &cm_id,
(void *)hca_ptr, RDMA_PS_TCP)) {
}
/* set inline max with env or default, get local lid and gid 0 */
- if (hca_ptr->ib_hca_handle->device->transport_type
- == IBV_TRANSPORT_IWARP)
+ if (hca_ptr->ib_hca_handle->device->transport_type == IBV_TRANSPORT_IWARP)
hca_ptr->ib_trans.max_inline_send =
dapl_os_get_env_val("DAPL_MAX_INLINE",
INLINE_SEND_IWARP_DEFAULT);
hca_ptr->ib_trans.max_cm_retries =
dapl_os_get_env_val("DAPL_MAX_CM_RETRIES", IB_CM_RETRIES);
- /* set default IB MTU */
- hca_ptr->ib_trans.mtu = dapl_ib_mtu(2048);
-
dat_status = dapli_ib_thread_init();
if (dat_status != DAT_SUCCESS)
return dat_status;
return DAT_INTERNAL_ERROR;
#endif
+done:
+ /* set default IB MTU */
+ hca_ptr->ib_trans.mtu = dapl_ib_mtu(2048);
+
return DAT_SUCCESS;
}
dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " close_hca: %p->%p\n",
hca_ptr, hca_ptr->ib_hca_handle);
+ if (!g_ib_thread_state) /* thread never started */
+ goto bail;
+
#ifdef DAT_IB_COLLECTIVES
dapli_free_collective_service(hca_ptr);
#endif
dapl_os_sleep_usec(1000);
}
bail:
-
if (hca_ptr->ib_trans.ib_cq)
ibv_destroy_comp_channel(hca_ptr->ib_trans.ib_cq);
#include "dapl_evd_util.h"
#include "dapl_ib_util.h"
#include "dapl_ep_util.h"
+#include "dapl_ia_util.h"
#include "dapl_cookie.h"
+#include "dapl_provider.h"
+
#include <stdarg.h>
#ifdef DAT_IB_COLLECTIVES
IN int op_type,
IN DAT_COMPLETION_FLAGS flags, IN DAT_IB_ADDR_HANDLE * ah);
+DAT_RETURN
+dapli_open_query_ext(IN const DAT_NAME_PTR name,
+ OUT DAT_IA_HANDLE * ia_handle,
+ IN DAT_IA_ATTR_MASK ia_mask,
+ OUT DAT_IA_ATTR * ia_attr,
+ IN DAT_PROVIDER_ATTR_MASK pr_mask,
+ OUT DAT_PROVIDER_ATTR * pr_attr);
+
/*
* dapl_extensions
*
*/
DAT_RETURN
dapl_extensions(IN DAT_HANDLE dat_handle,
- IN DAT_EXTENDED_OP ext_op, IN va_list args)
+ IN DAT_EXTENDED_OP ext_op,
+ IN va_list args)
{
DAT_EP_HANDLE ep;
DAT_IB_ADDR_HANDLE *ah = NULL;
switch ((int)ext_op) {
+ case DAT_IB_OPEN_QUERY_OP:
+ {
+ dapl_dbg_log(DAPL_DBG_TYPE_RTN,
+ " OPEN_QUERY extension call\n");
+
+ DAT_IA_HANDLE *ia_handle = va_arg(args, DAT_IA_HANDLE *);
+ DAT_IA_ATTR_MASK ia_mask = va_arg(args, DAT_IA_ATTR_MASK);
+ DAT_IA_ATTR *ia_attr = va_arg(args, DAT_IA_ATTR *);
+ DAT_PROVIDER_ATTR_MASK pr_mask = va_arg(args, DAT_PROVIDER_ATTR_MASK);
+ DAT_PROVIDER_ATTR *pr_attr = va_arg(args, DAT_PROVIDER_ATTR *);
+ DAT_NAME_PTR name = (DAT_NAME_PTR) dat_handle;
+
+ status = dapli_open_query_ext(name, ia_handle, ia_mask,
+ ia_attr, pr_mask, pr_attr);
+ break;
+ }
+ case DAT_IB_CLOSE_QUERY_OP:
+ dapl_dbg_log(DAPL_DBG_TYPE_RTN,
+ " CLOSE_QUERY extension call\n");
+
+ status = dapl_ia_close(dat_handle, DAT_CLOSE_ABRUPT_FLAG);
+ break;
+
case DAT_IB_RDMA_WRITE_IMMED_OP:
dapl_dbg_log(DAPL_DBG_TYPE_RTN,
" WRITE_IMMED_DATA extension call\n");
break;
}
}
+
+/*
+ * dapli_open_query_ext
+ *
+ *
+ * Direct link to provider for quick provider query without full IA device open
+ *
+ * Input:
+ * provider name
+ * ia_attr
+ * provider_attr
+ *
+ * Output:
+ * ia_attr
+ * provider_attr
+ *
+ * Return Values:
+ * DAT_SUCCESS
+ * DAT_INSUFFICIENT_RESOURCES
+ * DAT_INVALID_PARAMETER
+ * DAT_INVALID_HANDLE
+ * DAT_PROVIDER_NOT_FOUND (returned by dat registry if necessary)
+ */
+DAT_RETURN
+dapli_open_query_ext(IN const DAT_NAME_PTR name,
+ OUT DAT_IA_HANDLE * ia_handle_ptr,
+ IN DAT_IA_ATTR_MASK ia_mask,
+ OUT DAT_IA_ATTR * ia_attr,
+ IN DAT_PROVIDER_ATTR_MASK pr_mask,
+ OUT DAT_PROVIDER_ATTR * pr_attr)
+{
+ DAT_RETURN dat_status = DAT_SUCCESS;
+ DAT_PROVIDER *provider;
+ DAPL_HCA *hca_ptr = NULL;
+ DAT_IA_HANDLE ia_ptr = NULL;
+
+ dapl_log(DAPL_DBG_TYPE_EXTENSION,
+ "dapli_open_query_ext (%s, 0x%llx, %p, 0x%x, %p)\n",
+ name, ia_mask, ia_attr, pr_mask, pr_attr);
+
+ dat_status = dapl_provider_list_search(name, &provider);
+ if (DAT_SUCCESS != dat_status) {
+ dat_status = DAT_ERROR(DAT_INVALID_PARAMETER, DAT_INVALID_ARG1);
+ goto bail;
+ }
+
+ /* ia_handle_ptr and async_evd_handle_ptr cannot be NULL */
+ if ((ia_attr == NULL) && (pr_attr == NULL)) {
+ return DAT_ERROR(DAT_INVALID_PARAMETER, DAT_INVALID_ARG5);
+ }
+
+ /* initialize the caller's OUT param */
+ *ia_handle_ptr = DAT_HANDLE_NULL;
+
+ /* get the hca_ptr */
+ hca_ptr = (DAPL_HCA *) provider->extension;
+
+ /* log levels could be reset and set between open_query calls */
+ if (dapl_os_get_env_val("DAPL_DBG_TYPE", 0))
+ g_dapl_dbg_type = dapl_os_get_env_val("DAPL_DBG_TYPE", 0);
+
+ /*
+ * Open the HCA if it has not been done before.
+ */
+ dapl_os_lock(&hca_ptr->lock);
+ if (hca_ptr->ib_hca_handle == IB_INVALID_HANDLE) {
+ /* open in query mode */
+ dat_status = dapls_ib_open_hca(hca_ptr->name,
+ hca_ptr, DAPL_OPEN_QUERY);
+ if (dat_status != DAT_SUCCESS) {
+ dapl_dbg_log(DAPL_DBG_TYPE_ERR,
+ "dapls_ib_open_hca failed %x\n",
+ dat_status);
+ dapl_os_unlock(&hca_ptr->lock);
+ goto bail;
+ }
+ }
+ /* Take a reference on the hca_handle */
+ dapl_os_atomic_inc(&hca_ptr->handle_ref_count);
+ dapl_os_unlock(&hca_ptr->lock);
+
+ /* Allocate and initialize ia structure */
+ ia_ptr = (DAT_IA_HANDLE) dapl_ia_alloc(provider, hca_ptr);
+ if (!ia_ptr) {
+ dat_status = DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_MEMORY);
+ goto cleanup;
+ }
+
+ dat_status = dapl_ia_query(ia_ptr, NULL, ia_mask, ia_attr, pr_mask, pr_attr);
+ if (dat_status != DAT_SUCCESS) {
+ dapl_dbg_log(DAPL_DBG_TYPE_ERR,
+ "dapls_ib_query_hca failed %x\n", dat_status);
+ goto cleanup;
+ }
+
+ *ia_handle_ptr = ia_ptr;
+ return DAT_SUCCESS;
+
+cleanup:
+ /* close device and release HCA reference */
+ if (ia_ptr) {
+ dapl_ia_close(ia_ptr, DAT_CLOSE_ABRUPT_FLAG);
+ } else {
+ dapl_os_lock(&hca_ptr->lock);
+ dapls_ib_close_hca(hca_ptr);
+ hca_ptr->ib_hca_handle = IB_INVALID_HANDLE;
+ dapl_os_atomic_dec(&hca_ptr->handle_ref_count);
+ dapl_os_unlock(&hca_ptr->lock);
+ }
+bail:
+ return dat_status;
+}
+
}
}
#endif
+ dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
+ " mr_register: ia=%p, lmr=%p va=%p ln=%d return\n",
+ ia_ptr, lmr, virt_addr, length, privileges);
+
return DAT_SUCCESS;
}
}
}
-
+char *dapl_ib_port_str(enum ibv_port_state state)
+{
+ switch (state) {
+ case IBV_PORT_NOP:
+ return "NOP";
+ case IBV_PORT_DOWN:
+ return "DOWN";
+ case IBV_PORT_INIT:
+ return "INIT";
+ case IBV_PORT_ARMED:
+ return "ARMED";
+ case IBV_PORT_ACTIVE:
+ return "ACTIVE";
+ case IBV_PORT_ACTIVE_DEFER:
+ return "DEFER";
+ default:
+ return "UNKNOWN";
+ }
+}
/*
* dapls_ib_query_hca
struct ibv_device_attr dev_attr;
struct ibv_port_attr port_attr;
- if (hca_ptr->ib_hca_handle == NULL) {
- dapl_dbg_log(DAPL_DBG_TYPE_ERR, " query_hca: BAD handle\n");
- return (DAT_INVALID_HANDLE);
- }
-
/* local IP address of device, set during ia_open */
- if (ip_addr != NULL)
+ if (ip_addr)
memcpy(ip_addr, &hca_ptr->hca_address, sizeof(DAT_SOCK_ADDR6));
if (ia_attr == NULL && ep_attr == NULL)
return DAT_SUCCESS;
+ if (ia_attr != NULL) /* setup address ptr, even with no device */
+ ia_attr->ia_address_ptr = (DAT_IA_ADDRESS_PTR) &hca_ptr->hca_address;
+
+ if (hca_ptr->ib_hca_handle == NULL) /* no open device, query mode */
+ return DAT_SUCCESS;
+
/* query verbs for this device and port attributes */
if (ibv_query_device(hca_ptr->ib_hca_handle, &dev_attr) ||
ibv_query_port(hca_ptr->ib_hca_handle,
dapl_os_get_env_val("DAPL_MCM_MSG_MAX", DAT_MIX_RDMA_MAX));
}
#endif
+
if (ia_attr != NULL) {
(void)dapl_os_memzero(ia_attr, sizeof(*ia_attr));
strncpy(ia_attr->adapter_name,
/* set provider/transport specific named attributes */
hca_ptr->ib_trans.dev_str = ia_attr->adapter_name;
hca_ptr->ib_trans.mtu_str = dapl_ib_mtu_str(hca_ptr->ib_trans.mtu);
+ hca_ptr->ib_trans.port_state_str = dapl_ib_port_str(port_attr.state);
hca_ptr->ib_trans.guid = ntohll(ibv_get_device_guid(hca_ptr->ib_trans.ib_dev));
sprintf(hca_ptr->ib_trans.guid_str, "%04x:%04x:%04x:%04x",
(unsigned) (hca_ptr->ib_trans.guid >> 48) & 0xffff,
(unsigned) (hca_ptr->ib_trans.guid >> 0) & 0xffff);
#ifdef _OPENIB_MCM_
hca_ptr->ib_trans.sys_guid = dev_attr.sys_image_guid; /* network order */
- if (hca_ptr->ib_trans.scif_ep)
+ if (hca_ptr->ib_trans.self.node)
hca_ptr->ib_trans.mode_str = "PROXY";
else
hca_ptr->ib_trans.mode_str = "DIRECT";
dapl_log(DAPL_DBG_TYPE_UTIL,
" query_hca: msg %llu rdma %llu iov %d lmr %d rmr %d"
- " ack_time %d mr %u\n",
+ " ack_time %d mr %u ia_addr_ptr %p\n",
ia_attr->max_message_size, ia_attr->max_rdma_size,
ia_attr->max_iov_segments_per_dto,
ia_attr->max_lmrs, ia_attr->max_rmrs,
hca_ptr->ib_trans.ack_timer,
- ia_attr->max_lmr_block_size);
+ ia_attr->max_lmr_block_size,
+ ia_attr->ia_address_ptr);
}
if (ep_attr != NULL) {
{
"DAT_IB_TRANSPORT_MTU", "2048"}
,
+ {
+ "DAT_IB_PORT_STATUS", "UNKNOWN"}
+ ,
#ifdef DAT_EXTENSIONS
{
"DAT_EXTENSION_INTERFACE", "TRUE"}
void dapls_query_provider_specific_attr(IN DAPL_IA * ia_ptr,
IN DAT_PROVIDER_ATTR * attr_ptr)
{
+
attr_ptr->num_provider_specific_attr = SPEC_ATTR_SIZE(ib_attrs);
attr_ptr->provider_specific_attr = ib_attrs;
+ dapl_log(DAPL_DBG_TYPE_UTIL,
+ " prov_attr: %p sz %d\n", ib_attrs, SPEC_ATTR_SIZE(ib_attrs));
+
/* update common attributes from providers */
ib_attrs[1].value = ia_ptr->hca_ptr->ib_trans.dev_str;
ib_attrs[2].value = ia_ptr->hca_ptr->ib_trans.mode_str;
ib_attrs[3].value = ia_ptr->hca_ptr->ib_trans.read_str;
ib_attrs[4].value = ia_ptr->hca_ptr->ib_trans.guid_str;
ib_attrs[5].value = ia_ptr->hca_ptr->ib_trans.mtu_str;
+ ib_attrs[6].value = ia_ptr->hca_ptr->ib_trans.port_state_str;
+
}
/*
char *mtu_str;
char *mode_str;
char *read_str;
+ char *port_state_str;
} ib_hca_transport_t;
DAT_RETURN dapls_modify_qp_rtu(struct ibv_qp *qp, uint32_t qpn, uint16_t lid, ib_gid_handle_t gid);
/* MIC eXchange (MIX) operations */
-int dapli_mix_open(ib_hca_transport_t *tp, char *name, int port);
+int dapli_mix_open(ib_hca_transport_t *tp, char *name, int port, int query);
void dapli_mix_close(ib_hca_transport_t *tp);
int dapli_mix_listen(dp_ib_cm_handle_t cm, uint16_t sid);
int dapli_mix_listen_free(dp_ib_cm_handle_t cm);
* dapl_convert_errno
*
*/
-DAT_RETURN dapls_ib_open_hca(IN IB_HCA_NAME hca_name, IN DAPL_HCA * hca_ptr)
+DAT_RETURN dapls_ib_open_hca(IN IB_HCA_NAME hca_name,
+ IN DAPL_HCA * hca_ptr,
+ IN DAPL_OPEN_FLAGS flags)
{
struct ibv_device **dev_list;
- struct dat_mcm_addr *mcm_ia = &hca_ptr->hca_address;
+ struct dat_mcm_addr *mcm_ia = (struct dat_mcm_addr *) &hca_ptr->hca_address;
struct ibv_port_attr port_attr;
int i;
- DAT_RETURN dat_status;
+ DAT_RETURN dat_status = DAT_SUCCESS;
+
+ if (flags & DAPL_OPEN_QUERY) {
+ dapl_log(DAPL_DBG_TYPE_WARN,
+ " WARNING! open_hca: %s %s - %p in %s\n",
+ PROVIDER_NAME, hca_name, hca_ptr,
+ flags & DAPL_OPEN_QUERY ? "QUERY MODE":"");
+ }
/* Get list of all IB devices, find match, open */
dev_list = ibv_get_device_list(NULL);
hca_ptr->ib_trans.mtu =
dapl_ib_mtu(dapl_os_get_env_val("DAPL_IB_MTU", DCM_IB_MTU));
- if (dapli_mix_open(&hca_ptr->ib_trans, hca_name, hca_ptr->port_num)) {
+ if (dapli_mix_open(&hca_ptr->ib_trans, hca_name,
+ hca_ptr->port_num, flags & DAPL_OPEN_QUERY)) {
dapl_log(DAPL_DBG_TYPE_ERR,
" open_hca: SCIF init ERR for %s\n",
ibv_get_device_name(hca_ptr->ib_trans.ib_dev));
goto err;
}
+ if (flags & DAPL_OPEN_QUERY)
+ goto done;
+
/* initialize CM list, LISTEN, SND queue, PSP array, locks */
if ((dapl_os_lock_init(&hca_ptr->ib_trans.lock)) != DAT_SUCCESS)
goto err;
&hca_ptr->hca_address)->sin_addr),
mcm_map_str(hca_ptr->ib_trans.addr.ep_map));
+ /* wait for cm_thread */
+ while (hca_ptr->ib_trans.cm_state != IB_THREAD_RUN)
+ dapl_os_sleep_usec(1000);
+
+done:
/* save LID, GID, QPN, PORT address information, for ia_queries */
/* Set AF_INET6 to insure callee address storage of 28 bytes */
hca_ptr->ib_trans.hca = hca_ptr;
dapl_log(DAPL_DBG_TYPE_CM, " MCM IA: AF %d LID 0x%x QPN 0x%x GID"
" 0x" F64x ":" F64x " port %d ep_map %s sl %d qt %d\n",
- mcm_ia->family, ntohl(mcm_ia->qpn), ntohs(mcm_ia->lid),
+ mcm_ia->family, ntohs(mcm_ia->lid), ntohl(mcm_ia->qpn),
(unsigned long long)ntohll(*(uint64_t*)&mcm_ia->gid[0]),
(unsigned long long)ntohll(*(uint64_t*)&mcm_ia->gid[8]),
- mcm_ia->port, mcm_map_str(mcm_ia->ep_map), mcm_ia->sl, mcm_ia->qp_type);
+ mcm_ia->port, mcm_map_str(mcm_ia->ep_map),
+ mcm_ia->sl, mcm_ia->qp_type);
ibv_free_device_list(dev_list);
-
- /* wait for cm_thread */
- while (hca_ptr->ib_trans.cm_state != IB_THREAD_RUN)
- dapl_os_sleep_usec(1000);
-
return dat_status;
-
bail:
mcm_service_destroy(hca_ptr);
ibv_close_device(hca_ptr->ib_hca_handle);
hca_ptr->ib_hca_handle = IB_INVALID_HANDLE;
-
err:
ibv_free_device_list(dev_list);
return DAT_INTERNAL_ERROR;
{
dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " close_hca: %p\n", hca_ptr);
+ if (!hca_ptr->ib_trans.cm_state) /* thread never started */
+ goto done;
+
if (hca_ptr->ib_trans.cm_state == IB_THREAD_RUN) {
hca_ptr->ib_trans.cm_state = IB_THREAD_CANCEL;
dapls_thread_signal(&hca_ptr->ib_trans.signal);
}
dapli_mix_close(&hca_ptr->ib_trans);
-
dapl_os_lock_destroy(&hca_ptr->ib_trans.lock);
dapl_os_lock_destroy(&hca_ptr->ib_trans.llock);
destroy_os_signal(hca_ptr);
mcm_service_destroy(hca_ptr);
-
+done:
if (hca_ptr->ib_trans.ib_cq)
ibv_destroy_comp_channel(hca_ptr->ib_trans.ib_cq);
recv_wr.num_sge = 1;
sge.length = mlen + hlen;
sge.lkey = tp->mr_rbuf->lkey;
- rbuf = tp->rbuf;
+ rbuf = (char *) tp->rbuf;
for (i = 0; i < tp->qpe; i++) {
recv_wr.wr_id = (uintptr_t) (rbuf + hlen);
*
* MIX_IA_OPEN
*/
-int dapli_mix_open(ib_hca_transport_t *tp, char *name, int port)
+int dapli_mix_open(ib_hca_transport_t *tp, char *name, int port, int query_only)
{
int ret, len;
dat_mix_open_t msg;
}
dapl_log(DAPL_DBG_TYPE_EXTENSION," SCIF node_id: %d\n", (uint16_t)tp->self.node);
- if (tp->self.node == 0 && !always_proxy) {
+ if (query_only || (tp->self.node == 0 && !always_proxy)){
dapl_log(DAPL_DBG_TYPE_EXTENSION," Not running on MIC, no MPXY connect required\n");
tp->scif_ep = 0;
return 0;
char *mtu_str;
char *mode_str;
char *read_str;
+ char *port_state_str;
#ifdef DAT_IB_COLLECTIVES
/* Collective member device and address information */
ib_thread_state_t coll_thread_state;
* dapl_convert_errno
*
*/
-DAT_RETURN dapls_ib_open_hca(IN IB_HCA_NAME hca_name, IN DAPL_HCA * hca_ptr)
+DAT_RETURN dapls_ib_open_hca(IN IB_HCA_NAME hca_name,
+ IN DAPL_HCA * hca_ptr,
+ IN DAPL_OPEN_FLAGS flags)
{
struct ibv_device **dev_list;
struct ibv_port_attr port_attr;
int i;
- DAT_RETURN dat_status;
-
- dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
- " open_hca: %s - %p\n", hca_name, hca_ptr);
+ DAT_RETURN dat_status = DAT_SUCCESS;
- /* get the IP address of the device */
- dat_status = getlocalipaddr((char *)&hca_ptr->hca_address,
- sizeof(DAT_SOCK_ADDR6));
- if (dat_status != DAT_SUCCESS)
- return dat_status;
+ if (flags & DAPL_OPEN_QUERY) {
+ dapl_log(DAPL_DBG_TYPE_WARN,
+ " WARNING! open_hca: %s %s - %p in %s\n",
+ PROVIDER_NAME, hca_name, hca_ptr,
+ flags & DAPL_OPEN_QUERY ? "QUERY MODE":"");
+ }
-#ifdef DAPL_DBG
- /* DBG: unused port, set process id, lower 16 bits of pid */
- ((struct sockaddr_in *)&hca_ptr->hca_address)->sin_port =
- htons((uint16_t)dapl_os_getpid());
-#endif
/* Get list of all IB devices, find match, open */
dev_list = ibv_get_device_list(NULL);
if (!dev_list) {
hca_name))
goto found;
}
-
dapl_log(DAPL_DBG_TYPE_ERR,
" open_hca: device %s not found\n", hca_name);
goto err;
hca_ptr->ib_trans.mtu =
dapl_ib_mtu(dapl_os_get_env_val("DAPL_IB_MTU", SCM_IB_MTU));
+ if (flags & DAPL_OPEN_QUERY)
+ goto done;
+
+ /* get the IP address of the device */
+ dat_status = getlocalipaddr((char *)&hca_ptr->hca_address,
+ sizeof(DAT_SOCK_ADDR6));
+ if (dat_status != DAT_SUCCESS)
+ return dat_status;
+
+#ifdef DAPL_DBG
+ /* DBG: unused port, set process id, lower 16 bits of pid */
+ ((struct sockaddr_in *)&hca_ptr->hca_address)->sin_port =
+ htons((uint16_t)dapl_os_getpid());
+#endif
/* EVD events without direct CQ channels, CNO support */
hca_ptr->ib_trans.ib_cq =
dapl_os_sleep_usec(1000);
}
- dapl_log(DAPL_DBG_TYPE_CM,
- " SCM IA: devname %s, IB port %d, hostname_IP %s\n",
- ibv_get_device_name(hca_ptr->ib_trans.ib_dev),
- hca_ptr->port_num,
- inet_ntoa(((struct sockaddr_in *)
- &hca_ptr->hca_address)->sin_addr));
dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
" open_hca: LID 0x%x GID Subnet 0x" F64x " ID 0x" F64x
"\n", ntohs(hca_ptr->ib_trans.lid), (unsigned long long)
goto bail;
#endif
+done:
+ dapl_log(DAPL_DBG_TYPE_CM,
+ " SCM IA: devname %s, IB port %d, hostname_IP %s\n",
+ ibv_get_device_name(hca_ptr->ib_trans.ib_dev),
+ hca_ptr->port_num,
+ inet_ntoa(((struct sockaddr_in *)
+ &hca_ptr->hca_address)->sin_addr));
ibv_free_device_list(dev_list);
return dat_status;
- bail:
+bail:
ibv_close_device(hca_ptr->ib_hca_handle);
hca_ptr->ib_hca_handle = IB_INVALID_HANDLE;
- err:
+err:
ibv_free_device_list(dev_list);
return DAT_INTERNAL_ERROR;
}
{
dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " close_hca: %p\n", hca_ptr);
+ if (!g_ib_thread_state) /* thread never started */
+ goto out;
+
#ifdef DAT_IB_COLLECTIVES
dapli_free_collective_service(hca_ptr);
#endif
-
dapl_os_lock(&g_hca_lock);
if (g_ib_thread_state != IB_THREAD_RUN) {
dapl_os_unlock(&g_hca_lock);
char *mtu_str;
char *mode_str;
char *read_str;
+ char *port_state_str;
#ifdef DAT_IB_COLLECTIVES
/* Collective member device and address information */
ib_thread_state_t coll_thread_state;
* dapl_convert_errno
*
*/
-DAT_RETURN dapls_ib_open_hca(IN IB_HCA_NAME hca_name, IN DAPL_HCA * hca_ptr)
+DAT_RETURN dapls_ib_open_hca(IN IB_HCA_NAME hca_name,
+ IN DAPL_HCA * hca_ptr,
+ IN DAPL_OPEN_FLAGS flags)
{
struct ibv_device **dev_list;
- union dcm_addr *ucm_ia = &hca_ptr->hca_address;
+ union dcm_addr *ucm_ia = (union dcm_addr *) &hca_ptr->hca_address;
struct ibv_port_attr port_attr;
int i;
- DAT_RETURN dat_status;
+ DAT_RETURN dat_status = DAT_SUCCESS;
char gid_str[INET6_ADDRSTRLEN];
+ if (flags & DAPL_OPEN_QUERY) {
+ dapl_log(DAPL_DBG_TYPE_WARN,
+ " WARNING! open_hca: %s %s - %p in %s\n",
+ PROVIDER_NAME, hca_name, hca_ptr,
+ flags & DAPL_OPEN_QUERY ? "QUERY MODE":"");
+ }
+
/* Get list of all IB devices, find match, open */
dev_list = ibv_get_device_list(NULL);
if (!dev_list) {
hca_name))
goto found;
}
-
dapl_log(DAPL_DBG_TYPE_ERR,
" open_hca: device %s not found\n", hca_name);
goto err;
found:
-
hca_ptr->ib_hca_handle = ibv_open_device(hca_ptr->ib_trans.ib_dev);
if (!hca_ptr->ib_hca_handle) {
dapl_log(DAPL_DBG_TYPE_ERR,
hca_ptr->ib_trans.mtu =
dapl_ib_mtu(dapl_os_get_env_val("DAPL_IB_MTU", DCM_IB_MTU));
+ if (flags & DAPL_OPEN_QUERY)
+ goto done;
+
/* initialize CM list, LISTEN, SND queue, PSP array, locks */
if ((dapl_os_lock_init(&hca_ptr->ib_trans.lock)) != DAT_SUCCESS)
goto err;
inet_ntoa(((struct sockaddr_in *)
&hca_ptr->hca_address)->sin_addr));
+#ifdef DAT_IB_COLLECTIVES
+ if (dapli_create_collective_service(hca_ptr))
+ goto bail;
+#endif
+
+ /* wait for cm_thread */
+ while (hca_ptr->ib_trans.cm_state != IB_THREAD_RUN)
+ dapl_os_sleep_usec(1000);
+
+done:
/* save LID, GID, QPN, PORT address information, for ia_queries */
/* Set AF_INET6 to insure callee address storage of 28 bytes */
hca_ptr->ib_trans.hca = hca_ptr;
- hca_ptr->ib_trans.addr.ib.family = AF_INET6;
+ hca_ptr->ib_trans.addr.ib.family = AF_INET6;
hca_ptr->ib_trans.addr.ib.qp_type = IBV_QPT_UD;
- memcpy(&hca_ptr->hca_address,
- &hca_ptr->ib_trans.addr,
+ memcpy(&hca_ptr->hca_address,
+ &hca_ptr->ib_trans.addr,
sizeof(union dcm_addr));
dapl_log(DAPL_DBG_TYPE_CM, " UCM IA: AF %d LID 0x%x QPN 0x%x GID"
" 0x" F64x ":" F64x " port %d sl %d qt %d\n",
- ucm_ia->ib.family, ntohl(ucm_ia->ib.qpn), ntohs(ucm_ia->ib.lid),
+ ucm_ia->ib.family, ntohs(ucm_ia->ib.lid), ntohl(ucm_ia->ib.qpn),
(unsigned long long)ntohll(*(uint64_t*)&ucm_ia->ib.gid[0]),
(unsigned long long)ntohll(*(uint64_t*)&ucm_ia->ib.gid[8]),
ucm_ia->ib.port, ucm_ia->ib.sl, ucm_ia->ib.qp_type);
-#ifdef DAT_IB_COLLECTIVES
- if (dapli_create_collective_service(hca_ptr))
- goto bail;
-#endif
-
ibv_free_device_list(dev_list);
-
- /* wait for cm_thread */
- while (hca_ptr->ib_trans.cm_state != IB_THREAD_RUN)
- dapl_os_sleep_usec(1000);
-
return dat_status;
-
bail:
ucm_service_destroy(hca_ptr);
ibv_close_device(hca_ptr->ib_hca_handle);
hca_ptr->ib_hca_handle = IB_INVALID_HANDLE;
-
err:
ibv_free_device_list(dev_list);
return DAT_INTERNAL_ERROR;
{
dapl_dbg_log(DAPL_DBG_TYPE_UTIL, " close_hca: %p\n", hca_ptr);
+ if (!hca_ptr->ib_trans.cm_state) /* thread never started */
+ goto done;
+
#ifdef DAT_IB_COLLECTIVES
dapli_free_collective_service(hca_ptr);
#endif
dapl_os_lock_destroy(&hca_ptr->ib_trans.llock);
destroy_os_signal(hca_ptr);
ucm_service_destroy(hca_ptr);
-
+done:
if (hca_ptr->ib_trans.ib_cq)
ibv_destroy_comp_channel(hca_ptr->ib_trans.ib_cq);
recv_wr.num_sge = 1;
sge.length = mlen + hlen;
sge.lkey = tp->mr_rbuf->lkey;
- rbuf = tp->rbuf;
+ rbuf = (char *) tp->rbuf;
for (i = 0; i < tp->qpe; i++) {
recv_wr.wr_id = (uintptr_t) (rbuf + hlen);
#ifdef DAT_EXTENSIONS
extern int g_dat_extensions;
-
+extern DAT_RETURN udat_extension_open(IN const DAT_NAME_PTR name,
+ IN DAT_EXTENDED_OP ext_op,
+ IN va_list args);
+extern DAT_RETURN udat_extension_close(IN const DAT_NAME_PTR name,
+ IN DAT_EXTENDED_OP ext_op,
+ IN va_list args);
+
+/* Consumer API - dat_extension_op()
+ *
+ * Handle == IA, EP, EVD, etc
+ * !Handle == direct extension operation to provider without device open
+ * provider name supplied for linkage to library
+ *
+ */
DAT_RETURN DAT_API dat_extension_op(IN DAT_HANDLE handle,
IN DAT_EXTENDED_OP ext_op, IN ...)
{
DAT_IA_HANDLE dapl_handle;
va_list args;
- if (handle == NULL) {
- return DAT_ERROR(DAT_INVALID_HANDLE,
- DAT_INVALID_HANDLE1);
- }
+ dat_os_dbg_print(DAT_OS_DBG_TYPE_CONSUMER_API,
+ " dat_extension_op: (handle %p, op %d) called\n",
+ handle, ext_op);
/* If not IA handle then just passthrough */
- if (dats_get_ia_handle(handle, &dapl_handle) != DAT_SUCCESS) {
+ if (dats_get_ia_handle(handle, &dapl_handle) != DAT_SUCCESS)
dapl_handle = handle;
- }
- /* verify provider extension support */
- if (!g_dat_extensions) {
+ dat_os_dbg_print(DAT_OS_DBG_TYPE_CONSUMER_API,
+ " dat_extension_op: dapl_handle %p \n", handle);
+
+ /* verify provider extension support, if open */
+ if (dapl_handle && !g_dat_extensions)
return DAT_ERROR(DAT_NOT_IMPLEMENTED, 0);
- }
/* extension will validate the handle based on op */
va_start(args, ext_op);
- status = DAT_HANDLE_EXTENDEDOP(dapl_handle, ext_op, args);
- va_end(args);
+ if (ext_op & DAT_OPEN_EXTENSION_BASE) {
+ const DAT_NAME_PTR name = va_arg(args, const DAT_NAME_PTR);
+
+ dat_os_dbg_print(DAT_OS_DBG_TYPE_CONSUMER_API,
+ " call udat_ext_open: (name %p, %s op %d) called\n",
+ name, name, handle, ext_op);
+ if (name == NULL)
+ status = DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_ARG3);
+ else
+ status = udat_extension_open(name, ext_op, args);
+
+ } else if (ext_op & DAT_CLOSE_EXTENSION_BASE) {
+ dat_os_dbg_print(DAT_OS_DBG_TYPE_CONSUMER_API,
+ " call udat_ext_close: handle %p\n", handle);
+ status = udat_extension_close(handle, ext_op, args);
+ } else {
+ dat_os_dbg_print(DAT_OS_DBG_TYPE_CONSUMER_API,
+ " call dat_ext_op: handle %p\n", dapl_handle);
+ status = DAT_HANDLE_EXTENDEDOP(dapl_handle, ext_op, args);
+ }
+ va_end(args);
return status;
}
#endif
return status;
}
+//***********************************************************************
+// Function: dat_dr_provider_open_ext
+//***********************************************************************
+#ifdef DAT_EXTENSIONS
+DAT_RETURN
+dat_dr_provider_open_ext(IN const DAT_PROVIDER_INFO *info,
+ OUT DAT_HANDLE_EXTENDEDOP_FUNC *p_ext_func)
+{
+ DAT_RETURN status;
+ DAT_DICTIONARY_DATA data;
+
+ dat_os_lock(&g_dr_lock);
+ status = dat_dictionary_search(g_dr_dictionary, info, &data);
+ dat_os_unlock(&g_dr_lock);
+
+ if (DAT_SUCCESS == status) {
+ ((DAT_DR_ENTRY *) data)->ref_count++;
+ *p_ext_func = ((DAT_DR_ENTRY *)data)->ia_ext_func;
+ }
+
+ return status;
+}
+#endif
+
//***********************************************************************
// Function: dat_dr_provider_close
//***********************************************************************
DAT_COUNT ref_count;
DAT_IA_OPEN_FUNC ia_open_func;
DAT_PROVIDER_INFO info;
+#ifdef DAT_EXTENSIONS
+ DAT_HANDLE_EXTENDEDOP_FUNC ia_ext_func;
+#endif /* DAT_EXTENSIONS */
} DAT_DR_ENTRY;
dat_dr_remove (
IN const DAT_PROVIDER_INFO *info );
-
extern DAT_RETURN
dat_dr_provider_open (
IN const DAT_PROVIDER_INFO *info,
OUT DAT_IA_OPEN_FUNC *p_ia_open_func );
+#ifdef DAT_EXTENSIONS
+extern DAT_RETURN
+dat_dr_provider_open_ext (
+ IN const DAT_PROVIDER_INFO *info,
+ OUT DAT_HANDLE_EXTENDEDOP_FUNC *p_ext_func );
+#endif
+
extern DAT_RETURN
dat_dr_provider_close (
IN const DAT_PROVIDER_INFO *info);
IN DAT_COUNT); /* low_watermark */
#ifdef DAT_EXTENSIONS
+#define DAT_OPEN_EXTENSION_BASE 0x1000
+#define DAT_CLOSE_EXTENSION_BASE 0x2000
typedef int DAT_EXTENDED_OP;
extern DAT_RETURN DAT_API dat_extension_op(
IN DAT_HANDLE, /* handle */
* 2.0.5 - Add DAT_IB_UD extended UD connection error events
* 2.0.6 - Add MPI over IB collective extensions
* 2.0.7 - Add new IA counters for dapl CM, device LINK, device DIAG
+ * 2.0.8 - Add DAT_IB_OPEN_QUERY_OP, DAT_IB_CLOSE_QUERY_OP, fast provider query interface
*
*/
-#define DAT_IB_EXTENSION_VERSION 207 /* 2.0.7 */
+#define DAT_IB_EXTENSION_VERSION 208 /* 2.0.8 */
#define DAT_IB_ATTR_COUNTERS "DAT_COUNTERS"
#define DAT_IB_ATTR_FETCH_AND_ADD "DAT_IB_FETCH_AND_ADD"
#define DAT_IB_ATTR_CMP_AND_SWAP "DAT_IB_CMP_AND_SWAP"
DAT_IB_COLLECTIVE_BARRIER_OP,
DAT_IB_START_COUNTERS_OP,
DAT_IB_STOP_COUNTERS_OP,
+ /* OPEN and CLOSE extensions require DAT support, set proper range */
+ DAT_IB_OPEN_QUERY_OP = DAT_OPEN_EXTENSION_BASE,
+ DAT_IB_CLOSE_QUERY_OP = DAT_CLOSE_EXTENSION_BASE,
} DAT_IB_OP;
IN (DAT_IB_OP) DAT_IB_STOP_COUNTERS_OP, \
IN (DAT_COUNTER_TYPE) (type))
+/*
+ * dat_ib_open_query:
+ * dat_ib_close_query:
+ *
+ * Given IA name, open appropriate provider for fast attribute query
+ *
+ * OUT:
+ * ia_handle
+ * ia_attr
+ * pr_attr
+ *
+ * RETURN VALUE:
+ *
+ * DAT_SUCCESS
+ * DAT_INVALID_HANDLE
+ * DAT_NOT_IMPLEMENTED
+ *
+ */
+#define dat_ib_open_query(name, ia_handle, ia_mask, ia_attr, prov_mask, prov_attr) \
+ dat_extension_op(\
+ IN (DAT_HANDLE) (NULL), \
+ IN (DAT_IB_OP) DAT_IB_OPEN_QUERY_OP, \
+ IN (const DAT_NAME_PTR) (name), \
+ OUT (DAT_HANDLE *) (ia_handle), \
+ IN (DAT_IA_ATTR_MASK) (ia_mask), \
+ OUT (DAT_IA_ATTR *) (ia_attr), \
+ IN (DAT_PROVIDER_ATTR_MASK) (prov_mask), \
+ OUT (DAT_PROVIDER_ATTR *) (prov_attr))
+
+#define dat_ib_close_query(ia_handle) \
+ dat_extension_op(\
+ IN (DAT_HANDLE) (ia_handle), \
+ IN (DAT_IB_OP) DAT_IB_CLOSE_QUERY_OP)
+
/*
************************ MPI IB Collective Functions ***********************
*/
entry.ref_count = 0;
entry.ia_open_func = provider->ia_open_func;
entry.info = *provider_info;
-
+#ifdef DAT_EXTENSIONS
+ entry.ia_ext_func = provider->handle_extendedop_func;
+#endif
return dat_dr_insert(provider_info, &entry);
}
return status;
}
+#ifdef DAT_EXTENSIONS
+
+/***********************************************************************
+ * Function: udat_extension_open - provider name supplied
+ ***********************************************************************/
+DAT_RETURN DAT_API
+udat_extension_open(IN const DAT_NAME_PTR name,
+ IN DAT_EXTENDED_OP ext_op,
+ IN va_list args)
+{
+ DAT_HANDLE_EXTENDEDOP_FUNC prov_ext_func;
+ DAT_PROVIDER_INFO info;
+ DAT_OS_SIZE len;
+ DAT_RETURN dat_status;
+
+ dat_os_dbg_print(DAT_OS_DBG_TYPE_CONSUMER_API,
+ " udat_extenstion_op(%s,%x:%x,%x,%d) called\n",
+ name, ext_op, DAT_VERSION_MAJOR, DAT_VERSION_MINOR, DAT_THREADSAFE);
+
+ if (UDAT_IS_BAD_POINTER(name)) {
+ return DAT_ERROR(DAT_INVALID_PARAMETER, DAT_INVALID_ARG1);
+ }
+
+ len = dat_os_strlen(name);
+
+ if (DAT_NAME_MAX_LENGTH <= len) {
+ return DAT_ERROR(DAT_INVALID_PARAMETER, DAT_INVALID_ARG1);
+ }
+
+ if (DAT_FALSE == udat_check_state()) {
+ return DAT_ERROR(DAT_INVALID_STATE, 0);
+ }
+
+ dat_os_strncpy(info.ia_name, name, len + 1);
+
+ info.dapl_version_major = DAT_VERSION_MAJOR;
+ info.dapl_version_minor = DAT_VERSION_MINOR;
+ info.is_thread_safe = DAT_THREADSAFE;
+
+#ifndef DAT_NO_STATIC_REGISTRY
+ (void)dat_sr_provider_open(&info);
+#endif
+ dat_status = dat_dr_provider_open_ext(&info, &prov_ext_func);
+ if (dat_status != DAT_SUCCESS) {
+ dat_os_dbg_print(DAT_OS_DBG_TYPE_CONSUMER_API,
+ " DAT Registry: udat_ext_op () provider information "
+ "for IA name %s not found in dynamic registry\n",
+ name);
+ return dat_status;
+ }
+ g_dat_extensions = 1;
+ dat_status = (*prov_ext_func) (name, ext_op, args);
+
+ dat_os_dbg_print(DAT_OS_DBG_TYPE_CONSUMER_API,
+ " udat_extension_op () return = 0x%x for IA name %s\n",
+ dat_status, name);
+
+ if (dat_status != DAT_SUCCESS) {
+ (void)dat_dr_provider_close(&info);
+#ifndef DAT_NO_STATIC_REGISTRY
+ (void)dat_sr_provider_close(&info);
+#endif
+ }
+
+ return dat_status;
+}
+/***********************************************************************
+ * Function: udat_extension_close - provider handle supplied
+ ***********************************************************************/
+DAT_RETURN DAT_API
+udat_extension_close(IN DAT_IA_HANDLE ia_handle,
+ IN DAT_EXTENDED_OP ext_op,
+ IN va_list args)
+{
+ DAT_PROVIDER *provider;
+ DAT_PROVIDER_INFO info;
+ DAT_OS_SIZE len;
+ DAT_RETURN dat_status;
+ const char* ia_name;
+
+ dat_os_dbg_print(DAT_OS_DBG_TYPE_CONSUMER_API,
+ " udat_ext_close_op(ia=%p,op=%x) called\n",
+ ia_handle, ext_op);
+
+ if (ia_handle == NULL)
+ return DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_IA);
+
+ if (DAT_FALSE == udat_check_state())
+ return DAT_ERROR(DAT_INVALID_STATE, 0);
+
+ provider = DAT_HANDLE_TO_PROVIDER(ia_handle);
+ ia_name = provider->device_name;
+
+ dat_os_dbg_print(DAT_OS_DBG_TYPE_CONSUMER_API,
+ " udat_ext_close_op(ia=%p,op=%x,name=%s) called\n",
+ ia_handle, ext_op, ia_name);
+
+ dat_status = (*provider->handle_extendedop_func) (ia_handle, ext_op, args);
+
+ dat_os_dbg_print(DAT_OS_DBG_TYPE_CONSUMER_API,
+ " udat_extension_op () return = 0x%x for IA name %s\n",
+ dat_status, ia_name);
+
+ if (dat_status != DAT_SUCCESS)
+ return dat_status;
+
+ len = dat_os_strlen(ia_name);
+ dat_os_assert(len < DAT_NAME_MAX_LENGTH);
+ dat_os_strncpy(info.ia_name, ia_name, len + 1);
+ info.dapl_version_major = DAT_VERSION_MAJOR;
+ info.dapl_version_minor = DAT_VERSION_MINOR;
+ info.is_thread_safe = DAT_THREADSAFE;
+
+ dat_status = dat_dr_provider_close(&info);
+ if (DAT_SUCCESS != dat_status) {
+ dat_os_dbg_print(DAT_OS_DBG_TYPE_CONSUMER_API,
+ "udat_ext_close_op: dynamic registry unable to close "
+ "provider for IA name %s\n", ia_name);
+ }
+
+#ifndef DAT_NO_STATIC_REGISTRY
+ dat_status = dat_sr_provider_close(&info);
+ if (DAT_SUCCESS != dat_status) {
+ dat_os_dbg_print(DAT_OS_DBG_TYPE_CONSUMER_API,
+ "udat_ext_close_op: static registry unable to close "
+ "provider for IA name %s\n", ia_name);
+ }
+#endif
+ return dat_status;
+}
+#endif
+
/*
* Local variables:
* c-indent-level: 4
#define RDMA_BUFFER_SIZE (4*1024*1024)
/* Global DAT vars */
+static DAT_IA_HANDLE h_ia_query = DAT_HANDLE_NULL;
static DAT_IA_HANDLE h_ia = DAT_HANDLE_NULL;
static DAT_PZ_HANDLE h_pz = DAT_HANDLE_NULL;
static DAT_EP_HANDLE h_ep = DAT_HANDLE_NULL;
static int burst_msg_index = 0;
static int ucm = 0;
static int write_only = 1; /* only test rdma_writes by default */
+static int query_only = 0;
static int rq_cnt, sq_cnt;
static DAT_SOCK_ADDR6 remote;
{
char str[INET6_ADDRSTRLEN] = {" ??? "};
+ if (!sa) {
+ printf("%d Local Address not provided - port %d\n", getpid(), SERVER_CONN_QUAL);
+ return;
+ }
+
switch(sa->sa_family) {
case AF_INET:
inet_ntop(AF_INET, &((struct sockaddr_in *)sa)->sin_addr, str, INET6_ADDRSTRLEN);
DAT_PROVIDER_ATTR pr_attr;
/* parse arguments */
- while ((c = getopt(argc, argv, "Aautscvpq:l:b:d:B:h:P:S:e:")) != -1) {
+ while ((c = getopt(argc, argv, "AQautscvpq:l:b:d:B:h:P:S:e:")) != -1) {
switch (c) {
case 'A': /* all tests, msg, rdma read and write */
write_only = 0;
break;
+ case 'Q': /* query only */
+ query_only = 1;
+ break;
case 'a':
align_data = 1;
fflush(stdout);
LOGPRINTF("%d Allocated RMR buffers (r:%p,s:%p) len %d \n",
getpid(), p_rmr_rcv, p_rmr_snd, 4096);
+ /* query_by_name */
+ start = get_time();
+ ret = dat_ib_open_query(provider,
+ &h_ia_query,
+ DAT_IA_FIELD_ALL, &ia_attr,
+ DAT_PROVIDER_FIELD_PROVIDER_SPECIFIC_ATTR,
+ &pr_attr);
+ stop = get_time();
+ ts.open += ((stop - start) * 1.0e6);
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d: Error open_query: %s\n",
+ getpid(), DT_RetToStr(ret));
+ exit(1);
+ } else
+ LOGPRINTF("%d Query_by_name \n", getpid());
+
+ print_ia_address(ia_attr.ia_address_ptr);
+
+ /* Provider specific attributes */
+ for (i=0; i<pr_attr.num_provider_specific_attr; i++) {
+ LOGPRINTF("%d Provider_attr[%d] %s = %s \n",
+ getpid(), i,
+ pr_attr.provider_specific_attr[i].name,
+ pr_attr.provider_specific_attr[i].value);
+ }
+ ret = dat_ib_close_query(h_ia_query);
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d: Error close_query: %s\n",
+ getpid(), DT_RetToStr(ret));
+ exit(1);
+ } else
+ LOGPRINTF("%d Query_by_name \n", getpid());
+
+ if (query_only)
+ goto close;
+
/* dat_ia_open, dat_pz_create */
h_async_evd = DAT_HANDLE_NULL;
start = get_time();
getpid(), DT_RetToStr(ret));
exit(1);
}
+ LOGPRINTF("%d Query done. ia_attr.addr %p hw_ver %d ext_ver %d\n",
+ getpid(), ia_attr.ia_address_ptr,
+ ia_attr.hardware_version_major,
+ ia_attr.extension_version);
+
print_ia_address(ia_attr.ia_address_ptr);
/* Provider specific attributes */
LOGPRINTF("%d Freed pz\n", getpid());
h_pz = NULL;
}
-
+close:
/* close the device */
LOGPRINTF("%d Closing Interface Adaptor\n", getpid());
- start = get_time();
- ret = dat_ia_close(h_ia, DAT_CLOSE_ABRUPT_FLAG);
- stop = get_time();
- ts.close += ((stop - start) * 1.0e6);
- if (ret != DAT_SUCCESS) {
- fprintf(stderr, "%d: Error Adaptor close: %s\n",
- getpid(), DT_RetToStr(ret));
- } else
- LOGPRINTF("%d Closed Interface Adaptor\n", getpid());
+ if (h_ia) {
+ start = get_time();
+ ret = dat_ia_close(h_ia, DAT_CLOSE_ABRUPT_FLAG);
+ stop = get_time();
+ ts.close += ((stop - start) * 1.0e6);
+ if (ret != DAT_SUCCESS) {
+ fprintf(stderr, "%d: Error Adaptor close: %s\n",
+ getpid(), DT_RetToStr(ret));
+ } else
+ LOGPRINTF("%d Closed Interface Adaptor\n", getpid());
+ }
+
/* free rdma buffers */
free(rbuf);
projects / ~ardavis / dapl.git / commitdiff